Skip to content

Commit 7682890

Browse files
shahor02shahor02
committed
o2-raw-tf-reader supports remote TF reading (with --copy-cmd none)
The same conventions as for the o2-ctf-reader. But remote reading of raw files is very slow, makes sense only if a small fraction of data needs to be read.
1 parent 8c2216f commit 7682890

6 files changed

Lines changed: 345 additions & 158 deletions

File tree

‎Detectors/Raw/TFReaderDD/CMakeLists.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111

1212
o2_add_library(TFReaderDD
1313
SOURCES src/SubTimeFrameFile.cxx
14+
src/BinFileOp.cxx
1415
src/SubTimeFrameFileReader.cxx
1516
PUBLIC_LINK_LIBRARIES FairRoot::Base
1617
O2::Headers

‎Detectors/Raw/TFReaderDD/include/TFReaderDD/BinFileOp.h‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
// Copyright 2019-2020 CERN and copyright holders of ALICE O2.
2+
// See https://alice-o2.web.cern.ch/copyright for details of the copyright holders.
3+
// All rights not expressly granted are reserved.
4+
//
5+
// This software is distributed under the terms of the GNU General Public
6+
// License v3 (GPL Version 3), copied verbatim in the file "COPYING".
7+
//
8+
// In applying this license CERN does not waive the privileges and immunities
9+
// granted to it by virtue of its status as an Intergovernmental Organization
10+
// or submit itself to any jurisdiction.
11+
12+
// @brief Polymorphic class to access either local or grid files with fixed sed ot methods
13+
#ifndef _BIN_FILE_OP_H_
14+
#define _BIN_FILE_OP_H_
15+
16+
#include <TFile.h>
17+
#include <boost/filesystem.hpp>
18+
#include <boost/iostreams/device/mapped_file.hpp>
19+
#include <string>
20+
#include <algorithm>
21+
22+
namespace o2::rawdd
23+
{
24+
25+
class BinFileOp
26+
{
27+
public:
28+
static constexpr size_t KBYTE = 1024;
29+
static constexpr size_t MaxBuffSize = KBYTE << 2;
30+
virtual ~BinFileOp() = default;
31+
32+
virtual bool read_advance(void* ptr, size_t len) = 0;
33+
virtual void set_position(size_t pos) = 0;
34+
virtual bool ignore_nbytes(size_t pLen) = 0;
35+
virtual bool isGood() const = 0;
36+
virtual unsigned char* bufferize(size_t& s) = 0;
37+
virtual size_t bufferized_size() const = 0;
38+
size_t bufferized_pos() const { return mBufferizedPos; }
39+
size_t size() const { return mFileSize; }
40+
size_t position() const { return mFileOffset; }
41+
size_t distance_to_eof() const { return mFileSize - mFileOffset; }
42+
bool eof() const { return mFileOffset == mFileSize; }
43+
44+
static BinFileOp* open(const std::string& name);
45+
46+
protected:
47+
BinFileOp(const std::string& name) : mFileName(name) {}
48+
49+
std::string mFileName = {};
50+
size_t mFileOffset = 0;
51+
size_t mFileSize = 0;
52+
size_t mBufferizedPos = -1UL;
53+
};
54+
55+
//========================================================================
56+
class BinFileOpLocal : public BinFileOp
57+
{
58+
public:
59+
BinFileOpLocal(const std::string& name);
60+
~BinFileOpLocal() override;
61+
62+
bool read_advance(void* ptr, size_t len) override;
63+
void set_position(size_t pos) override
64+
{
65+
assert(pos <= mFileSize);
66+
mFileOffset = std::min(pos, mFileSize);
67+
}
68+
unsigned char* bufferize(size_t& s) override;
69+
size_t bufferized_size() const override { return mFileSize - mBufferizedPos; }
70+
71+
bool ignore_nbytes(size_t len) override;
72+
bool isGood() const override { return mFileMap.is_open(); }
73+
74+
size_t size() const { return mFileSize; }
75+
size_t position() const { return mFileOffset; }
76+
bool eof() const { return mFileOffset == mFileSize; }
77+
78+
protected:
79+
boost::iostreams::mapped_file_source mFileMap;
80+
};
81+
82+
//========================================================================
83+
class BinFileOpGrid : public BinFileOp
84+
{
85+
public:
86+
BinFileOpGrid(const std::string& name);
87+
~BinFileOpGrid() override = default;
88+
89+
bool read_advance(void* ptr, size_t len) override;
90+
unsigned char* bufferize(size_t& s) override;
91+
size_t bufferized_size() const override { return mBuffer.size(); }
92+
void set_position(size_t pos) override;
93+
bool ignore_nbytes(size_t len) override;
94+
bool isGood() const override { return mFile && !mFile->IsZombie(); }
95+
96+
size_t size() const { return mFileSize; }
97+
size_t position() const { return mFileOffset; }
98+
bool eof() const { return mFileOffset == mFileSize; }
99+
100+
protected:
101+
std::unique_ptr<TFile> mFile;
102+
std::vector<unsigned char> mBuffer;
103+
};
104+
105+
} // namespace o2::rawdd
106+
107+
#endif

‎Detectors/Raw/TFReaderDD/include/TFReaderDD/SubTimeFrameFileReader.h‎

Lines changed: 18 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,10 @@
2222
#include <fairmq/Parts.h>
2323
#include <fairmq/Device.h>
2424
#include <Framework/OutputRoute.h>
25-
#include <boost/filesystem.hpp>
26-
#include <boost/iostreams/device/mapped_file.hpp>
25+
#include "TFReaderDD/BinFileOp.h"
26+
// RSREM
27+
// #include <boost/filesystem.hpp>
28+
// #include <boost/iostreams/device/mapped_file.hpp>
2729
#include <fstream>
2830
#include <vector>
2931
#include <unordered_map>
@@ -47,34 +49,20 @@ class SubTimeFrameFileReader
4749

4850
SubTimeFrameFileReader() = delete;
4951
SubTimeFrameFileReader(const std::string& pFileName, o2::detectors::DetID::mask_t detMask, int verb, bool sup0xccdb, bool repaireHeaders, bool rejectDistSTF);
50-
~SubTimeFrameFileReader();
52+
~SubTimeFrameFileReader() = default;
5153

5254
/// Read a single TF from the file
5355
std::unique_ptr<MessagesPerRoute> read(fair::mq::Device* device, const std::vector<o2f::OutputRoute>& outputRoutes, const std::string& rawChannel, size_t slice);
5456

55-
/// Tell the current position of the file
56-
inline std::uint64_t position() const { return mFileMapOffset; }
57-
58-
/// Set the current position of the file
59-
inline void set_position(std::uint64_t pPos)
60-
{
61-
const std::uint64_t lPos = std::min(pPos, mFileSize);
62-
assert(pPos == lPos);
63-
mFileMapOffset = lPos;
64-
}
65-
66-
/// Is the stream position at EOF
67-
inline bool eof() const { return mFileMapOffset == mFileSize; }
68-
69-
/// Tell the size of the file
70-
inline std::uint64_t size() const { return mFileSize; }
71-
7257
private:
73-
std::string mFileName;
7458
std::unordered_map<o2::header::DataOrigin, bool> mDetOrigMap;
75-
boost::iostreams::mapped_file_source mFileMap;
76-
std::uint64_t mFileMapOffset = 0;
77-
std::uint64_t mFileSize = 0;
59+
60+
std::unique_ptr<BinFileOp> mFile;
61+
// RSREM
62+
// std::string mFileName;
63+
// boost::iostreams::mapped_file_source mFileMap;
64+
// std::uint64_t mFileMapOffset = 0;
65+
// std::uint64_t mFileSize = 0;
7866

7967
int mVerbosity = 0;
8068
bool mSup0xccdb = true;
@@ -84,55 +72,19 @@ class SubTimeFrameFileReader
8472
const std::string describeHeader(const o2::header::DataHeader& hd, bool full = false) const;
8573

8674
// helper to make sure written chunks are buffered, only allow pointers
87-
template <typename pointer,
88-
typename = std::enable_if_t<std::is_pointer<pointer>::value>>
89-
bool read_advance(pointer pPtr, std::uint64_t pLen)
75+
template <typename pointer, typename = std::enable_if_t<std::is_pointer<pointer>::value>>
76+
inline bool read_advance(pointer pPtr, std::uint64_t pLen)
9077
{
91-
if (!mFileMap.is_open()) {
78+
if (!mFile) {
9279
return false;
9380
}
94-
95-
assert(mFileMapOffset <= mFileSize);
96-
const std::uint64_t lToRead = std::min(pLen, mFileSize - mFileMapOffset);
97-
98-
if (lToRead != pLen) {
99-
LOGP(error, "FileReader: request to read beyond the file end. pos={} size={} len={}",
100-
mFileMapOffset, mFileSize, pLen);
101-
LOGP(error, "Closing the file {}. The read data is invalid.", mFileName);
102-
mFileMap.close();
103-
mFileMapOffset = 0;
104-
mFileSize = 0;
105-
return false;
106-
}
107-
108-
std::memcpy(reinterpret_cast<char*>(pPtr), mFileMap.data() + mFileMapOffset, lToRead);
109-
mFileMapOffset += lToRead;
110-
return true;
81+
return mFile->read_advance(pPtr, pLen);
11182
}
11283

11384
// return the pointer
114-
unsigned char* peek() const
115-
{
116-
return const_cast<unsigned char*>(reinterpret_cast<const unsigned char*>(mFileMap.data() + mFileMapOffset));
117-
}
85+
// inline unsigned char* peek() { mFile->peek(); }
11886

119-
inline bool ignore_nbytes(const std::size_t pLen)
120-
{
121-
const std::size_t lToIgnore = std::min(pLen, std::size_t(mFileSize - mFileMapOffset));
122-
if (pLen != lToIgnore) {
123-
LOGP(error, "FileReader: request to ignore bytes beyond the file end. pos={} size={} len={}",
124-
mFileMapOffset, mFileSize, pLen);
125-
LOGP(error, "Closing the file {}. The read data is invalid.", mFileName);
126-
mFileMap.close();
127-
mFileMapOffset = 0;
128-
mFileSize = 0;
129-
return false;
130-
}
131-
132-
mFileMapOffset += lToIgnore;
133-
assert(mFileMapOffset <= mFileSize);
134-
return true;
135-
}
87+
// inline bool ignore_nbytes(const std::size_t pLen) { mFle->ignore_nbytes(pLen); }
13688

13789
std::size_t getHeaderStackSize();
13890
o2::header::Stack getHeaderStack(std::size_t& pOrigsize);

0 commit comments

Comments
 (0)