Extended DEN format

Added an extension to a DEN format, where the sizes of dimensions might be uint32_t instead of uint16_t
File is in a DEN extended format if the first 6 bytes are zero
Next four bytes are uint32_t dimy
Next four bytes are uint32_t dimx
Next four bytes are uint32_t dimz

include/DEN/DenAsyncFrame2DWritter.hpp

@@ -20,6 +20,8 @@ namespace io {
     private:
         std::string projectionsFile;
         uint32_t sizex, sizey, sizez;
+        uint64_t offset;
+        bool extended;
         uint8_t* buffer;
         mutable std::mutex writingMutex;
 
@@ -70,12 +72,19 @@ namespace io {
             LOGE << msg;
             throw std::runtime_error(msg);
         }
+        extended = false;
+        offset = 6;
+        if(dimx > 65535 || dimy > 65535 || dimz > 65535)
+        {
+            extended = true;
+            offset = 18;
+        }
         this->projectionsFile = projectionsFile;
         this->sizex = dimx;
         this->sizey = dimy;
         this->sizez = dimz;
         uint64_t elementByteSize = sizeof(T);
-        uint64_t totalFileSize = uint64_t(6) + elementByteSize * dimx * dimy * dimz;
+        uint64_t totalFileSize = offset + elementByteSize * dimx * dimy * dimz;
         if(io::pathExists(projectionsFile))
         {
             uint64_t fileSize = io::getFileSize(projectionsFile);
@@ -91,16 +100,28 @@ namespace io {
         } else
         {
             io::createEmptyFile(projectionsFile, totalFileSize, true);
-        /*
-	    LOGD << io::xprintf("New file %s of the size %ld bytes was created.",
-                                projectionsFile.c_str(), totalFileSize);
-	*/
+            /*
+                LOGD << io::xprintf("New file %s of the size %ld bytes was created.",
+                                    projectionsFile.c_str(), totalFileSize);
+            */
+        }
+        uint8_t buf[18];
+        if(extended)
+        {
+            util::putUint16(0, &buf[0]);
+            util::putUint16(0, &buf[2]);
+            util::putUint16(0, &buf[4]);
+            util::putUint32(dimy, &buf[0]);
+            util::putUint32(dimx, &buf[2]);
+            util::putUint32(dimz, &buf[4]);
+            io::writeFirstBytes(projectionsFile, buf, 18);
+        } else
+        {
+            util::putUint16((uint16_t)dimy, &buf[0]);
+            util::putUint16((uint16_t)dimx, &buf[2]);
+            util::putUint16((uint16_t)dimz, &buf[4]);
+            io::writeFirstBytes(projectionsFile, buf, 6);
         }
-        uint8_t buf[6];
-        util::putUint16((uint16_t)dimy, &buf[0]);
-        util::putUint16((uint16_t)dimx, &buf[2]);
-        util::putUint16((uint16_t)dimz, &buf[4]);
-        io::writeFirstBytes(projectionsFile, buf, 6);
         buffer = new uint8_t[sizeof(T) * this->dimx() * this->dimy()];
     }
 
@@ -132,6 +153,8 @@ namespace io {
             this->sizex = b.dimx;
             this->sizey = b.dimy;
             this->sizez = b.dimz;
+            this->offset = offset;
+            this->extended = extended;
             if(this->buffer != nullptr)
             {
                 delete[] this->buffer;
@@ -150,6 +173,8 @@ namespace io {
         this->sizex = b.dimx;
         this->sizey = b.dimy;
         this->sizez = b.dimz;
+        this->offset = b.offset;
+        this->extended = b.extended;
         this->buffer = b.buffer;
         b.buffer = nullptr;
     }
@@ -165,6 +190,8 @@ namespace io {
             this->sizex = b.dimx;
             this->sizey = b.dimy;
             this->sizez = b.dimz;
+            this->offset = b.offset;
+            this->extended = b.extended;
             if(this->buffer != nullptr)
             {
                 delete[] this->buffer;
@@ -211,7 +238,7 @@ namespace io {
                 util::setNextElement<T>(s(k, j), &buffer[(j * this->dimx() + k) * sizeof(T)]);
             }
         }
-        uint64_t position = uint64_t(6) + ((uint64_t)sizeof(T)) * i * this->dimx() * this->dimy();
+        uint64_t position = offset + ((uint64_t)sizeof(T)) * i * this->dimx() * this->dimy();
         io::writeBytesFrom(projectionsFile, position, buffer,
                            sizeof(T) * this->dimx() * this->dimy());
         return;

include/DEN/DenFileInfo.hpp

@@ -18,14 +18,16 @@ namespace io {
     {
     public:
         DenFileInfo(std::string fileName);
-        uint16_t dimx() const;
-        uint16_t dimy() const;
-        uint16_t dimz() const;
-        uint16_t getNumRows() const;
-        uint16_t getNumCols() const;
-        uint16_t getNumSlices() const;
+        uint32_t dimx() const;
+        uint32_t dimy() const;
+        uint32_t dimz() const;
+        uint32_t getNumRows() const;
+        uint32_t getNumCols() const;
+        uint32_t getNumSlices() const;
         uint64_t getSize() const;
         uint64_t getNumPixels() const;
+        bool isExtended() const;
+        uint64_t getOffset() const;
         DenSupportedType getDataType() const;
         uint8_t elementByteSize() const;
         template <typename T>
@@ -37,6 +39,9 @@ namespace io {
 
     private:
         std::string fileName;
+		//Extended format means that the file has header 0,0,0 followed by the four dimensions so that the offset is 18 bytes and dimensions are stored in 6..9, 10..13 and 14..17 bytes as uint32_t
+		bool extended = false;
+		uint64_t offset = 6;
     };
 
     template <typename T>
@@ -47,7 +52,6 @@ namespace io {
         uint64_t cols = getNumCols();
         uint64_t zdim = getNumSlices();
         uint64_t currentPosition;
-        uint64_t offset = 6;
         switch(dataType)
         {
         case io::DenSupportedType::uint16_t_:
@@ -126,7 +130,6 @@ namespace io {
         uint64_t cols = getNumCols();
         uint64_t zdim = getNumSlices();
         uint64_t currentPosition;
-        uint64_t offset = 6;
         switch(dataType)
         {
         case io::DenSupportedType::uint16_t_:
@@ -211,7 +214,6 @@ namespace io {
         uint64_t dim_y = dimy();
         uint64_t dim_z = dimz();
         uint64_t currentPosition;
-        uint64_t offset = 6;
         double sum = 0.0;
         double val;
         switch(dataType)

include/DEN/DenFrame2DReader.hpp

@@ -46,12 +46,14 @@ namespace io {
         DenFrame2DReader<T>& operator=(DenFrame2DReader<T>&& other);
         std::shared_ptr<io::Frame2DI<T>> readFrame(unsigned int i) override;
         std::shared_ptr<io::BufferedFrame2D<T>> readBufferedFrame(unsigned int i);
-        unsigned int dimx() const override;
-        unsigned int dimy() const override;
-        unsigned int dimz() const override;
+        void readFrameIntoBuffer(unsigned int frameID, T* outside_buffer);
+        uint32_t dimx() const override;
+        uint32_t dimy() const override;
+        uint32_t dimz() const override;
         std::string getFileName() const;
         /**Returns file name of the underlying DEN file.**/
     private:
+		uint64_t offset;
         mutable std::mutex consistencyMutex;
     };
 
@@ -63,6 +65,7 @@ namespace io {
         this->sizex = pi.dimx();
         this->sizey = pi.dimy();
         this->sizez = pi.dimz();
+        this->offset = pi.getOffset();
         this->dataType = pi.getDataType();
         this->elementByteSize = pi.elementByteSize();
         this->buffer = new uint8_t[elementByteSize * sizex * sizey];
@@ -110,6 +113,7 @@ namespace io {
             this->sizey = b.sizey;
             this->sizex = b.sizex;
             this->sizez = b.sizez;
+            this->offset = b.offset;
             this->dataType = b.dataType;
             this->elementByteSize = b.elementByteSize;
             if(this->buffer != nullptr)
@@ -136,6 +140,7 @@ namespace io {
         this->sizex = b.sizex;
         this->sizey = b.sizey;
         this->sizez = b.sizez;
+            this->offset = b.offset;
         this->dataType = b.dataType;
         this->elementByteSize = b.elementByteSize;
         this->buffer = b.buffer;
@@ -165,6 +170,7 @@ namespace io {
             this->sizex = other.sizex;
             this->sizey = other.sizey;
             this->sizez = other.sizez;
+            this->offset = other.offset;
             this->dataType = other.dataType;
             this->elementByteSize = other.elementByteSize;
         }
@@ -178,19 +184,19 @@ namespace io {
     }
 
     template <typename T>
-    unsigned int DenFrame2DReader<T>::dimx() const
+    uint32_t DenFrame2DReader<T>::dimx() const
     {
         return sizex;
     }
 
     template <typename T>
-    unsigned int DenFrame2DReader<T>::dimy() const
+    uint32_t DenFrame2DReader<T>::dimy() const
     {
         return sizey;
     }
 
     template <typename T>
-    unsigned int DenFrame2DReader<T>::dimz() const
+    uint32_t DenFrame2DReader<T>::dimz() const
     {
         return sizez;
     }
@@ -209,7 +215,7 @@ namespace io {
         std::lock_guard<std::mutex> guard(consistencyMutex);
         // Mutex will be released as this goes out of scope.
         // To protect calling this method from another thread using the same block of memory
-        uint64_t position = uint64_t(6) + uint64_t(sliceNum) * elementByteSize * sizex * sizey;
+        uint64_t position = this->offset + uint64_t(sliceNum) * elementByteSize * sizex * sizey;
         io::readBytesFrom(this->denFile, position, buffer, elementByteSize * sizex * sizey);
         for(int a = 0; a != sizex * sizey; a++)
         {
@@ -219,5 +225,19 @@ namespace io {
             = std::make_shared<BufferedFrame2D<T>>(buffer_copy, sizex, sizey);
         return ps;
     }
+
+    template <typename T>
+    void DenFrame2DReader<T>::readFrameIntoBuffer(unsigned int frameID, T* outside_buffer)
+    {
+        std::lock_guard<std::mutex> guard(consistencyMutex);
+        // Mutex will be released as this goes out of scope.
+        // To protect calling this method from another thread using the same block of memory
+        uint64_t position = this->offset + uint64_t(frameID) * elementByteSize * sizex * sizey;
+        io::readBytesFrom(this->denFile, position, buffer, elementByteSize * sizex * sizey);
+        for(int a = 0; a != sizex * sizey; a++)
+        {
+            outside_buffer[a] = util::getNextElement<T>(&buffer[a * elementByteSize], dataType);
+        }
+    }
 } // namespace io
 } // namespace CTL

include/DEN/DenProjectionMatrixReader.hpp

@@ -18,8 +18,9 @@ namespace io {
     class DenProjectionMatrixReader : ProjectionMatrixReaderI
     {
     private:
-        int countMatrices;
         std::string projectionMatrixFile;
+        uint32_t countMatrices;
+		uint64_t offset;
 
     public:
         DenProjectionMatrixReader(std::string projectionMatrixFile);
@@ -27,7 +28,7 @@ namespace io {
         /*Returns i-th projection slice in the source.*/
         matrix::ProjectionMatrix readMatrix(int i) override;
         /**Number of projection matrices in the source.*/
-        unsigned int count() const override;
+        uint32_t count() const override;
     };
 } // namespace io
 } // namespace CTL

include/DEN/DenProjectionNakedReader.hpp

@@ -70,14 +70,16 @@ namespace io {
         /**
          * Access to the slice of the DEN file that is ordered as in the file.
          */
-        unsigned int dimx() const override;
-        unsigned int dimy() const override;
-        unsigned int dimz() const override;
+        uint32_t dimx() const override;
+        uint32_t dimy() const override;
+        uint32_t dimz() const override;
 
         std::string getProjectionsFile() { return projectionsFile; }
 
     private:
         T* buffer;
+		uint64_t offsetMatrix = 6;
+		uint64_t offsetProjections = 6;
     };
 
     template <typename T>
@@ -92,6 +94,8 @@ namespace io {
         this->sizey = pi.getNumRows();
         this->sizex = pi.getNumCols();
         this->sizez = pi.getNumSlices();
+		this->offsetMatrix = mi.getOffset();
+		this->offsetProjections = mi.getOffset();
         int cols, rows, matCount;
         cols = mi.getNumCols(); // Its matrix, dealing with strange data format considerations
         rows = mi.getNumRows(); // Its matrix, dealing with strange data format considerations
@@ -124,19 +128,19 @@ namespace io {
     }
 
     template <typename T>
-    unsigned int DenProjectionNakedReader<T>::dimx() const
+    uint32_t DenProjectionNakedReader<T>::dimx() const
     {
         return sizex;
     }
 
     template <typename T>
-    unsigned int DenProjectionNakedReader<T>::dimy() const
+    uint32_t DenProjectionNakedReader<T>::dimy() const
     {
         return sizey;
     }
 
     template <typename T>
-    unsigned int DenProjectionNakedReader<T>::dimz() const
+    uint32_t DenProjectionNakedReader<T>::dimz() const
     {
         return sizez;
     }
@@ -160,7 +164,7 @@ namespace io {
     DenProjectionNakedReader<T>::readProjectionMatrix(int sliceNum)
     {
         uint8_t buffer[8 * 3 * 4];
-        uint64_t position = ((uint64_t)6) + ((uint64_t)sliceNum) * 3 * 4 * 8;
+        uint64_t position = this->offsetMatrix + ((uint64_t)sliceNum) * 3 * 4 * 8;
         io::readBytesFrom(this->projectionMatrixFile, position, buffer, 8 * 3 * 4);
         double matrixData[3 * 4];
         for(int a = 0; a != 3 * 4; a++)
@@ -178,7 +182,7 @@ namespace io {
     template <typename T>
     std::shared_ptr<io::Frame2DI<T>> DenProjectionNakedReader<T>::readProjectionSlice(int sliceNum)
     {
-        uint64_t position = uint64_t(6) + uint64_t(sliceNum) * elementByteSize * sizex * sizey;
+        uint64_t position = this->offsetProjections + uint64_t(sliceNum) * elementByteSize * sizex * sizey;
         uint8_t* tmp = new uint8_t[elementByteSize * sizex * sizey];
         io::readBytesFrom(this->projectionsFile, position, tmp, elementByteSize * sizex * sizey);
         for(int a = 0; a != sizex * sizey; a++)

include/DEN/DenProjectionReader.hpp

@@ -23,11 +23,13 @@ namespace io {
     // ProjectionReaderI will be only once in the family tree
     {
     protected: // Visible in inheritance structure
-        int sizex, sizey, sizez;
+        uint32_t sizex, sizey, sizez;
         std::string projectionsFile;
         std::string projectionMatrixFile;
         DenSupportedType dataType;
-        int elementByteSize;
+        uint32_t elementByteSize;
+		uint64_t offsetMatrix = 6;
+		uint64_t offsetProjections = 6;
 
     public:
         DenProjectionReader(std::string projectionsFile, std::string projectionMatrixFile);
@@ -42,9 +44,9 @@ namespace io {
          * Access to the slice of the DEN file that is ordered as in the file.
          */
         void* readNakedMemory(int sliceNum);
-        unsigned int dimx() const override;
-        unsigned int dimy() const override;
-        unsigned int dimz() const override;
+        uint32_t dimx() const override;
+        uint32_t dimy() const override;
+        uint32_t dimz() const override;
     };
 
     template <typename T>
@@ -55,10 +57,12 @@ namespace io {
         this->projectionMatrixFile = projectionMatrixFile;
         DenFileInfo pi = DenFileInfo(this->projectionsFile);
         DenFileInfo mi = DenFileInfo(this->projectionMatrixFile);
-        this->sizey = pi.getNumRows();
-        this->sizex = pi.getNumCols();
-        this->sizez = pi.getNumSlices();
-        int cols, rows, matCount;
+        this->sizex = pi.dimx();
+        this->sizey = pi.dimy();
+        this->sizez = pi.dimz();
+		this->offsetMatrix = mi.getOffset();
+		this->offsetProjections = mi.getOffset();
+        uint32_t cols, rows, matCount;
         cols = mi.getNumCols(); // Its matrix, dealing with strange data format considerations
         rows = mi.getNumRows(); // Its matrix, dealing with strange data format considerations
         matCount = mi.getNumSlices();
@@ -123,7 +127,7 @@ namespace io {
     DenProjectionReader<T>::readProjectionMatrix(int sliceNum)
     {
         uint8_t buffer[8 * 3 * 4];
-        uint64_t position = ((uint64_t)6) + ((uint64_t)sliceNum) * 3 * 4 * 8;
+        uint64_t position = this->offsetMatrix + ((uint64_t)sliceNum) * 3 * 4 * 8;
         io::readBytesFrom(this->projectionMatrixFile, position, buffer, 8 * 3 * 4);
         double matrixData[3 * 4];
         for(int a = 0; a != 3 * 4; a++)
@@ -139,7 +143,7 @@ namespace io {
     std::shared_ptr<io::Frame2DI<T>> DenProjectionReader<T>::readProjectionSlice(int sliceNum)
     {
         uint8_t* buffer = new uint8_t[elementByteSize * sizex * sizey];
-        uint64_t position = (uint64_t)6 + ((uint64_t)sliceNum) * elementByteSize * sizex * sizey;
+        uint64_t position = this->offsetProjections + ((uint64_t)sliceNum) * elementByteSize * sizex * sizey;
         io::readBytesFrom(this->projectionsFile, position, buffer, elementByteSize * sizex * sizey);
         T* buffer_copy = new T[sizex * sizey];
         for(int a = 0; a != sizex * sizey; a++)
@@ -157,7 +161,7 @@ namespace io {
     void* DenProjectionReader<T>::readNakedMemory(int sliceNum)
     {
         uint8_t* buffer = new uint8_t[elementByteSize * sizex * sizey];
-        uint64_t position = (uint64_t)6 + ((uint64_t)sliceNum) * elementByteSize * sizex * sizey;
+        uint64_t position = this->offsetProjections + ((uint64_t)sliceNum) * elementByteSize * sizex * sizey;
         io::readBytesFrom(this->projectionsFile, position, buffer, elementByteSize * sizex * sizey);
         return buffer;
     }

include/DEN/DenSupportedType.hpp

@@ -26,7 +26,7 @@ namespace io {
 
 #pragma GCC diagnostic push
 #pragma GCC diagnostic ignored "-Wreturn-type"
-    inline DenSupportedType getSupportedTypeByByteLength(int byteLength)
+    inline DenSupportedType getSupportedTypeByByteLength(uint32_t byteLength)
     {
         switch(byteLength)
         {

include/Frame2DReaderI.hpp

@@ -19,11 +19,11 @@ namespace io {
     public:
         virtual std::shared_ptr<io::Frame2DI<T>> readFrame(unsigned int i) = 0;
         /*Returns i-th projection slice in the source.*/
-        virtual unsigned int dimx() const = 0;
+        virtual uint32_t dimx() const = 0;
         /**Returns x dimension.*/
-        virtual unsigned int dimy() const = 0;
+        virtual uint32_t dimy() const = 0;
         /**Returns y dimension.*/
-        virtual unsigned int dimz() const = 0;
+        virtual uint32_t dimz() const = 0;
         /**Returns number of slices in the source, slices are indexed 0 <= i < count().*/
     };
 } // namespace io

src/DenFileInfo.cpp

@@ -6,40 +6,78 @@ namespace io {
     DenFileInfo::DenFileInfo(std::string fileName)
         : fileName(fileName)
     {
+        uint8_t buffer[6];
+        readBytesFrom(this->fileName, 0, buffer, 6);
+        if(util::nextUint16(buffer) == 0 && util::nextUint16(buffer + 2) == 0
+           && util::nextUint16(buffer + 4) == 0 && getSize() > 6)
+        {
+            extended = true;
+            offset = 18;
+        } else
+        {
+            extended = false;
+            offset = 6;
+        }
     }
 
+    uint64_t DenFileInfo::getOffset() const { return offset; }
+    bool DenFileInfo::isExtended() const { return extended; }
     /**X dimension*/
-    uint16_t DenFileInfo::dimx() const
+    uint32_t DenFileInfo::dimx() const
     {
-        uint8_t buffer[2];
-        readBytesFrom(this->fileName, 2, buffer, 2);
-        return (util::nextUint16(buffer));
+        if(extended)
+        {
+            uint8_t buffer[4];
+            readBytesFrom(this->fileName, 6 + 4, buffer, 4);
+            return (util::nextUint32(buffer));
+        } else
+        {
+            uint8_t buffer[2];
+            readBytesFrom(this->fileName, 2, buffer, 2);
+            return (util::nextUint16(buffer));
+        }
     }
 
     /**Y dimension*/
-    uint16_t DenFileInfo::dimy() const
+    uint32_t DenFileInfo::dimy() const
     {
-        uint8_t buffer[2];
-        readBytesFrom(this->fileName, 0, buffer, 2);
-        return (util::nextUint16(buffer));
+        if(extended)
+        {
+            uint8_t buffer[4];
+            readBytesFrom(this->fileName, 6, buffer, 4);
+            return (util::nextUint32(buffer));
+        } else
+        {
+            uint8_t buffer[2];
+            readBytesFrom(this->fileName, 0, buffer, 2);
+            return (util::nextUint16(buffer));
+        }
     }
 
     /**Z dimension*/
-    uint16_t DenFileInfo::dimz() const
+    uint32_t DenFileInfo::dimz() const
     {
-        uint8_t buffer[2];
-        readBytesFrom(this->fileName, 4, buffer, 2);
-        return (util::nextUint16(buffer));
+        if(extended)
+        {
+            uint8_t buffer[4];
+            readBytesFrom(this->fileName, 6 + 8, buffer, 4);
+            return (util::nextUint32(buffer));
+        } else
+        {
+            uint8_t buffer[2];
+            readBytesFrom(this->fileName, 4, buffer, 2);
+            return (util::nextUint16(buffer));
+        }
     }
 
     /**Y dimension*/
-    uint16_t DenFileInfo::getNumRows() const { return this->dimy(); }
+    uint32_t DenFileInfo::getNumRows() const { return this->dimy(); }
 
     /**X dimension*/
-    uint16_t DenFileInfo::getNumCols() const { return this->dimx(); }
+    uint32_t DenFileInfo::getNumCols() const { return this->dimx(); }
 
     /**Z dimension*/
-    uint16_t DenFileInfo::getNumSlices() const { return this->dimz(); }
+    uint32_t DenFileInfo::getNumSlices() const { return this->dimz(); }
 
     /**File size
      */
@@ -81,7 +119,14 @@ namespace io {
 
     uint8_t DenFileInfo::elementByteSize() const
     {
-        uint64_t dataSize = this->getSize() - 6;
+        uint64_t dataSize;
+        if(extended)
+        {
+            dataSize = this->getSize() - 18;
+        } else
+        {
+            dataSize = this->getSize() - 6;
+        }
         uint64_t numPixels = this->getNumPixels();
         if(dataSize == 0)
         {

src/DenProjectionMatrixReader.cpp

@@ -6,7 +6,8 @@ namespace io {
     {
         this->projectionMatrixFile = projectionMatrixFile;
         DenFileInfo mi = DenFileInfo(this->projectionMatrixFile);
-        int cols, rows;
+		this->offset = mi.getOffset();
+        uint32_t cols, rows;
         cols = mi.getNumCols(); // Its matrix, dealing with strange data format considerations
         rows = mi.getNumRows(); // Its matrix, dealing with strange data format considerations
         countMatrices = mi.getNumSlices();
@@ -29,7 +30,7 @@ namespace io {
     matrix::ProjectionMatrix DenProjectionMatrixReader::readMatrix(int i)
     {
         uint8_t buffer[8 * 3 * 4];
-        uint64_t position = ((uint64_t)6) + ((uint64_t)i) * 3 * 4 * 8;
+        uint64_t position = this->offset + ((uint64_t)i) * 3 * 4 * 8;
         io::readBytesFrom(this->projectionMatrixFile, position, buffer, 8 * 3 * 4);
         double matrixData[3 * 4];
         for(int a = 0; a != 3 * 4; a++)
@@ -39,7 +40,7 @@ namespace io {
         return matrix::ProjectionMatrix(matrixData);
     }
 
-    unsigned int DenProjectionMatrixReader::count() const { return countMatrices; }
+    uint32_t DenProjectionMatrixReader::count() const { return countMatrices; }
 
 } // namespace io
 } // namespace CTL