diff --git a/src/bin/exrmetrics/exrmetrics.cpp b/src/bin/exrmetrics/exrmetrics.cpp index d3985272b..7c5839882 100644 --- a/src/bin/exrmetrics/exrmetrics.cpp +++ b/src/bin/exrmetrics/exrmetrics.cpp @@ -35,19 +35,20 @@ using IMATH_NAMESPACE::Box2i; using std::cerr; using namespace std::chrono; -using std::chrono::steady_clock; using std::cout; using std::endl; using std::list; +using std::min; using std::runtime_error; using std::string; using std::to_string; using std::vector; +using std::chrono::steady_clock; double timing (steady_clock::time_point start, steady_clock::time_point end) { - return std::chrono::duration(end-start).count(); + return std::chrono::duration (end - start).count (); } int @@ -63,8 +64,13 @@ channelCount (const Header& h) return channels; } -void -copyScanLine (InputPart& in, OutputPart& out) +// allocate pixelData and FrameBuffer to read scanline data from Input +partSizeData +initScanLine ( + vector>& pixelData, + FrameBuffer& buf, + InputPart& in, + const Header& outHeader) { Box2i dw = in.header ().dataWindow (); uint64_t width = dw.max.x + 1 - dw.min.x; @@ -72,19 +78,19 @@ copyScanLine (InputPart& in, OutputPart& out) uint64_t numPixels = width * height; int numChans = channelCount (in.header ()); - vector> pixelData (numChans); + pixelData.resize (numChans); uint64_t offsetToOrigin = width * static_cast (dw.min.y) + static_cast (dw.min.x); - int channelNumber = 0; - int pixelSize = 0; - FrameBuffer buf; + int channelNumber = 0; + int pixelSize = 0; - for (ChannelList::ConstIterator i = out.header ().channels ().begin (); - i != out.header ().channels ().end (); + for (ChannelList::ConstIterator i = outHeader.channels ().begin (); + i != outHeader.channels ().end (); ++i) { int samplesize = pixelTypeSize (i.channel ().type); + pixelSize += samplesize; pixelData[channelNumber].resize (numPixels * samplesize); buf.insert ( @@ -95,28 +101,48 @@ copyScanLine (InputPart& in, OutputPart& out) samplesize, samplesize * width)); ++channelNumber; - pixelSize += samplesize; } - in.setFrameBuffer (buf); - out.setFrameBuffer (buf); + partSizeData data; + data.rawSize = width * height * pixelSize; + data.pixelCount = width * height; + data.partType = in.header ().type (); + data.compression = in.header ().compression (); + data.channelCount = numChans; + return data; +} - steady_clock::time_point startRead = steady_clock::now(); +void +readScanLine (InputPart& in, FrameBuffer& buf, vector& perf) +{ + steady_clock::time_point start = steady_clock::now (); + Box2i dw = in.header ().dataWindow (); + in.setFrameBuffer (buf); in.readPixels (dw.min.y, dw.max.y); - steady_clock::time_point endRead = steady_clock::now(); - - steady_clock::time_point startWrite = steady_clock::now(); - out.writePixels (height); - steady_clock::time_point endWrite = steady_clock::now(); - - cout << " \"read time\": " << timing (startRead, endRead) << ",\n"; - cout << " \"write time\": " << timing (startWrite, endWrite) << ",\n"; - cout << " \"pixel count\": " << numPixels << ",\n"; - cout << " \"raw size\": " << numPixels * pixelSize << ",\n"; + steady_clock::time_point end = steady_clock::now (); + perf.push_back (timing (start, end)); } void -copyTiled (TiledInputPart& in, TiledOutputPart& out) +writeScanLine (OutputPart& out, FrameBuffer& buf, vector* perf) +{ + Box2i dw = out.header ().dataWindow (); + out.setFrameBuffer (buf); + steady_clock::time_point start = steady_clock::now (); + out.writePixels (dw.max.y - dw.min.y + 1); + if (perf) + { + steady_clock::time_point end = steady_clock::now (); + perf->push_back (timing (start, end)); + } +} + +partSizeData +initTiled ( + vector>>& pixelData, + vector& buf, + TiledInputPart& in, + const Header& outHeader) { int numChans = channelCount (in.header ()); TileDescription tiling = in.header ().tileDescription (); @@ -131,16 +157,15 @@ copyTiled (TiledInputPart& in, TiledOutputPart& out) totalLevels = in.numXLevels () * in.numYLevels (); break; case NUM_LEVELMODES: - default: - throw runtime_error ("unknown tile mode"); + default: throw runtime_error ("unknown tile mode"); } + pixelData.resize (totalLevels); + buf.resize (totalLevels); - vector>> pixelData (totalLevels); - vector frameBuffer (totalLevels); - - int levelIndex = 0; - int pixelSize = 0; - size_t totalPixels = 0; + int levelIndex = 0; + int64_t pixelSize = 0; + size_t totalPixels = 0; + int tileCount = 0; // // allocate memory and initialize frameBuffer for each level @@ -166,20 +191,19 @@ copyTiled (TiledInputPart& in, TiledOutputPart& out) width * static_cast (dw.min.y) + static_cast (dw.min.x); int channelNumber = 0; - pixelSize = 0; pixelData[levelIndex].resize (numChans); - + pixelSize = 0; for (ChannelList::ConstIterator i = - out.header ().channels ().begin (); - i != out.header ().channels ().end (); + outHeader.channels ().begin (); + i != outHeader.channels ().end (); ++i) { int samplesize = pixelTypeSize (i.channel ().type); pixelData[levelIndex][channelNumber].resize ( numPixels * samplesize); - frameBuffer[levelIndex].insert ( + buf[levelIndex].insert ( i.name (), Slice ( i.channel ().type, @@ -193,11 +217,26 @@ copyTiled (TiledInputPart& in, TiledOutputPart& out) totalPixels += numPixels; ++levelIndex; } + tileCount += in.numXTiles (xLevel) * in.numYTiles (yLevel); } } + partSizeData data; + data.rawSize = pixelSize * totalPixels; + data.pixelCount = totalPixels; + data.tileCount = tileCount; + data.isTiled = true; + data.partType = in.header ().type (); + data.compression = in.header ().compression (); + data.channelCount = numChans; + return data; +} - steady_clock::time_point startRead = steady_clock::now(); - levelIndex = 0; +void +readTiled (TiledInputPart& in, vector& buf, vector& perf) +{ + TileDescription tiling = in.header ().tileDescription (); + int levelIndex = 0; + steady_clock::time_point start = steady_clock::now (); for (int xLevel = 0; xLevel < in.numXLevels (); ++xLevel) { @@ -205,7 +244,7 @@ copyTiled (TiledInputPart& in, TiledOutputPart& out) { if (tiling.mode == RIPMAP_LEVELS || xLevel == yLevel) { - in.setFrameBuffer (frameBuffer[levelIndex]); + in.setFrameBuffer (buf[levelIndex]); in.readTiles ( 0, in.numXTiles (xLevel) - 1, @@ -218,70 +257,90 @@ copyTiled (TiledInputPart& in, TiledOutputPart& out) } } - steady_clock::time_point endRead = steady_clock::now(); + steady_clock::time_point end = steady_clock::now (); + perf.push_back (timing (start, end)); +} + +void +writeTiled ( + TiledOutputPart& out, vector& buf, vector* perf) +{ + int levelIndex = 0; - steady_clock::time_point startWrite = steady_clock::now(); - levelIndex = 0; - int tileCount = 0; + TileDescription tiling = out.header ().tileDescription (); + steady_clock::time_point start = steady_clock::now (); - for (int xLevel = 0; xLevel < in.numXLevels (); ++xLevel) + for (int xLevel = 0; xLevel < out.numXLevels (); ++xLevel) { - for (int yLevel = 0; yLevel < in.numYLevels (); ++yLevel) + for (int yLevel = 0; yLevel < out.numYLevels (); ++yLevel) { if (tiling.mode == RIPMAP_LEVELS || xLevel == yLevel) { - out.setFrameBuffer (frameBuffer[levelIndex]); + out.setFrameBuffer (buf[levelIndex]); out.writeTiles ( 0, - in.numXTiles (xLevel) - 1, + out.numXTiles (xLevel) - 1, 0, - in.numYTiles (yLevel) - 1, + out.numYTiles (yLevel) - 1, xLevel, yLevel); - tileCount += in.numXTiles (xLevel) * in.numYTiles (yLevel); ++levelIndex; } } } - steady_clock::time_point endWrite = steady_clock::now(); - - cout << " \"read time\": " << timing (startRead, endRead) << ",\n"; - cout << " \"write time\": " << timing (startWrite, endWrite) << ",\n"; - cout << " \"total tiles\": " << tileCount << ",\n"; - cout << " \"pixel count\": " << totalPixels << ",\n"; - cout << " \"raw size\": " << totalPixels * pixelSize << ",\n"; + if (perf) + { + steady_clock::time_point end = steady_clock::now (); + perf->push_back (timing (start, end)); + } } -void -copyDeepScanLine (DeepScanLineInputPart& in, DeepScanLineOutputPart& out) +// +// allocate arrays, assign frame buffer, read sample counts and +// main buffer. +// +// Also used to allocate buffers for re-reading. +// In that case, the inputSampleCount stores the per-pixel counts, +// so no need to do actual reading (performance counters not updated) +// +partSizeData +initAndReadDeepScanLine ( + vector& sampleCount, + vector>& sampleData, + vector>& pixelPtrs, + DeepFrameBuffer& buf, + DeepScanLineInputPart& in, + const vector* inputSampleCount, + const Header& outHeader, + vector& countPerf, + vector& samplePerf) { - Box2i dw = in.header ().dataWindow (); - uint64_t width = dw.max.x + 1 - dw.min.x; - uint64_t height = dw.max.y + 1 - dw.min.y; - uint64_t numPixels = width * height; - int numChans = channelCount (in.header ()); - vector sampleCount (numPixels); + Box2i dw = in.header ().dataWindow (); + uint64_t width = dw.max.x + 1 - dw.min.x; + uint64_t height = dw.max.y + 1 - dw.min.y; + uint64_t numPixels = width * height; + int numChans = channelCount (in.header ()); + sampleCount.resize (numPixels); uint64_t offsetToOrigin = width * static_cast (dw.min.y) + static_cast (dw.min.x); - vector> pixelPtrs (numChans); - DeepFrameBuffer buffer; + pixelPtrs.resize (numChans); - buffer.insertSampleCountSlice (Slice ( + buf.insertSampleCountSlice (Slice ( UINT, (char*) (sampleCount.data () - offsetToOrigin), sizeof (int), sizeof (int) * width)); int channelNumber = 0; int bytesPerSample = 0; - for (ChannelList::ConstIterator i = out.header ().channels ().begin (); - i != out.header ().channels ().end (); + for (ChannelList::ConstIterator i = outHeader.channels ().begin (); + i != outHeader.channels ().end (); ++i) { pixelPtrs[channelNumber].resize (numPixels); int samplesize = pixelTypeSize (i.channel ().type); - buffer.insert ( + buf.insert ( i.name (), DeepSlice ( i.channel ().type, @@ -293,21 +352,27 @@ copyDeepScanLine (DeepScanLineInputPart& in, DeepScanLineOutputPart& out) bytesPerSample += samplesize; } - in.setFrameBuffer (buffer); - out.setFrameBuffer (buffer); + const vector& samples = inputSampleCount ? *inputSampleCount + : sampleCount; + if (!inputSampleCount) + { + in.setFrameBuffer (buf); + + steady_clock::time_point startCountRead = steady_clock::now (); + in.readPixelSampleCounts (dw.min.y, dw.max.y); + steady_clock::time_point endCountRead = steady_clock::now (); - steady_clock::time_point startCountRead = steady_clock::now(); - in.readPixelSampleCounts (dw.min.y, dw.max.y); - steady_clock::time_point endCountRead = steady_clock::now(); + countPerf.push_back (timing (startCountRead, endCountRead)); + } size_t totalSamples = 0; - for (int i: sampleCount) + for (int i: samples) { totalSamples += i; } - vector> sampleData (numChans); + sampleData.resize (numChans); channelNumber = 0; for (ChannelList::ConstIterator i = in.header ().channels ().begin (); i != in.header ().channels ().end (); @@ -320,34 +385,83 @@ copyDeepScanLine (DeepScanLineInputPart& in, DeepScanLineOutputPart& out) { pixelPtrs[channelNumber][p] = sampleData[channelNumber].data () + offset * samplesize; - offset += sampleCount[p]; + offset += samples[p]; } ++channelNumber; } - steady_clock::time_point startSampleRead = steady_clock::now(); - in.readPixels (dw.min.y, dw.max.y); - steady_clock::time_point endSampleRead = steady_clock::now(); + if (!inputSampleCount) + { + + steady_clock::time_point startSampleRead = steady_clock::now (); + in.readPixels (dw.min.y, dw.max.y); + steady_clock::time_point endSampleRead = steady_clock::now (); + samplePerf.push_back (timing (startSampleRead, endSampleRead)); + } + + partSizeData data; + data.pixelCount = numPixels; + //raw size includes the sample count table + data.rawSize = totalSamples * bytesPerSample + numPixels * sizeof (int); + data.isDeep = true; + data.partType = in.header ().type (); + data.compression = in.header ().compression (); + data.channelCount = numChans; + + return data; +} + +void +readDeepScanLine ( + DeepScanLineInputPart& in, + DeepFrameBuffer& buf, + vector& samplePerf, + vector& countPerf) +{ - steady_clock::time_point startWrite = steady_clock::now(); - out.writePixels (height); - steady_clock::time_point endWrite = steady_clock::now(); + in.setFrameBuffer (buf); + Box2i dw = in.header ().dataWindow (); + steady_clock::time_point startCountRead = steady_clock::now (); + in.readPixelSampleCounts (dw.min.y, dw.max.y); + steady_clock::time_point endCountRead = steady_clock::now (); + countPerf.push_back (timing (startCountRead, endCountRead)); - cout << " \"count read time\": " << timing (startCountRead, endCountRead) - << ",\n"; - cout << " \"sample read time\": " - << timing (startSampleRead, endSampleRead) << ",\n"; - cout << " \"write time\": " << timing (startWrite, endWrite) << ",\n"; - cout << " \"pixel count\": " << numPixels << ",\n"; - cout << " \"raw size\": " - << totalSamples * bytesPerSample + numPixels * sizeof (int) << ",\n"; + steady_clock::time_point startSampleRead = steady_clock::now (); + in.readPixels (dw.min.y, dw.max.y); + steady_clock::time_point endSampleRead = steady_clock::now (); + samplePerf.push_back (timing (startSampleRead, endSampleRead)); } void -copyDeepTiled (DeepTiledInputPart& in, DeepTiledOutputPart& out) +writeDeepScanLine ( + DeepScanLineOutputPart& out, DeepFrameBuffer& buf, vector* perf) +{ + out.setFrameBuffer (buf); + Box2i dw = out.header ().dataWindow (); + + steady_clock::time_point start = steady_clock::now (); + out.writePixels (dw.max.y - dw.min.y + 1); + if (perf) + { + steady_clock::time_point end = steady_clock::now (); + perf->push_back (timing (start, end)); + } +} + +partSizeData +initAndReadDeepTiled ( + vector& sampleCount, + vector>& sampleData, + vector>& pixelPtrs, + DeepFrameBuffer& buf, + DeepTiledInputPart& in, + const vector* inputSampleCount, + const Header& outHeader, + vector& countPerf, + vector& samplePerf) { TileDescription tiling = in.header ().tileDescription (); @@ -364,33 +478,33 @@ copyDeepTiled (DeepTiledInputPart& in, DeepTiledOutputPart& out) "exrmetrics does not support ripmapped deep tiled parts"); } - Box2i dw = in.header ().dataWindow (); - uint64_t width = dw.max.x + 1 - dw.min.x; - uint64_t height = dw.max.y + 1 - dw.min.y; - uint64_t numPixels = width * height; - int numChans = channelCount (in.header ()); - vector sampleCount (numPixels); + Box2i dw = in.header ().dataWindow (); + uint64_t width = dw.max.x + 1 - dw.min.x; + uint64_t height = dw.max.y + 1 - dw.min.y; + uint64_t numPixels = width * height; + int numChans = channelCount (in.header ()); uint64_t offsetToOrigin = width * static_cast (dw.min.y) + static_cast (dw.min.x); - vector> pixelPtrs (numChans); - DeepFrameBuffer buffer; + pixelPtrs.resize (numChans); + sampleCount.resize (numPixels); - buffer.insertSampleCountSlice (Slice ( + buf.insertSampleCountSlice (Slice ( UINT, (char*) (sampleCount.data () - offsetToOrigin), sizeof (int), sizeof (int) * width)); int channelNumber = 0; int bytesPerSample = 0; - for (ChannelList::ConstIterator i = out.header ().channels ().begin (); - i != out.header ().channels ().end (); + + for (ChannelList::ConstIterator i = outHeader.channels ().begin (); + i != outHeader.channels ().end (); ++i) { pixelPtrs[channelNumber].resize (numPixels); int samplesize = pixelTypeSize (i.channel ().type); - buffer.insert ( + buf.insert ( i.name (), DeepSlice ( i.channel ().type, @@ -402,25 +516,31 @@ copyDeepTiled (DeepTiledInputPart& in, DeepTiledOutputPart& out) bytesPerSample += samplesize; } - in.setFrameBuffer (buffer); - out.setFrameBuffer (buffer); + const vector& samples = inputSampleCount ? *inputSampleCount + : sampleCount; - steady_clock::time_point startCountRead = steady_clock::now(); + if (!inputSampleCount) + { + in.setFrameBuffer (buf); - in.readPixelSampleCounts ( - 0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); - steady_clock::time_point endCountRead = steady_clock::now(); + steady_clock::time_point startCountRead = steady_clock::now (); + in.readPixelSampleCounts ( + 0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); + steady_clock::time_point endCountRead = steady_clock::now (); + countPerf.push_back (timing (startCountRead, endCountRead)); + } size_t totalSamples = 0; - for (int i: sampleCount) + for (int i: samples) { totalSamples += i; } - vector> sampleData (numChans); + sampleData.resize (numChans); channelNumber = 0; + for (ChannelList::ConstIterator i = in.header ().channels ().begin (); i != in.header ().channels ().end (); ++i) @@ -432,157 +552,642 @@ copyDeepTiled (DeepTiledInputPart& in, DeepTiledOutputPart& out) { pixelPtrs[channelNumber][p] = sampleData[channelNumber].data () + offset * samplesize; - offset += sampleCount[p]; + offset += samples[p]; } ++channelNumber; } - steady_clock::time_point startSampleRead = steady_clock::now(); + if (!inputSampleCount) + { + steady_clock::time_point startSampleRead = steady_clock::now (); + in.readTiles (0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); + steady_clock::time_point endSampleRead = steady_clock::now (); + + samplePerf.push_back (timing (startSampleRead, endSampleRead)); + } + + partSizeData data; + data.rawSize = totalSamples * bytesPerSample + numPixels * sizeof (int); + data.pixelCount = numPixels; + data.isDeep = true; + data.isTiled = true; + data.channelCount = numChans; + return data; +} + +void +readDeepTiled ( + DeepTiledInputPart& in, + DeepFrameBuffer& buf, + vector& countPerf, + vector& samplePerf) +{ + in.setFrameBuffer (buf); + + steady_clock::time_point startCountRead = steady_clock::now (); + + in.readPixelSampleCounts ( + 0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); + steady_clock::time_point endCountRead = steady_clock::now (); + + countPerf.push_back (timing (startCountRead, endCountRead)); + + steady_clock::time_point startSampleRead = steady_clock::now (); in.readTiles (0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); - steady_clock::time_point endSampleRead = steady_clock::now(); + steady_clock::time_point endSampleRead = steady_clock::now (); + + samplePerf.push_back (timing (startSampleRead, endSampleRead)); +} + +void +writeDeepTiled ( + DeepTiledOutputPart& out, DeepFrameBuffer& buf, vector* perf) +{ + out.setFrameBuffer (buf); + steady_clock::time_point startWrite = steady_clock::now (); + out.writeTiles (0, out.numXTiles (0) - 1, 0, out.numYTiles (0) - 1, 0, 0); + + if (perf) + { + steady_clock::time_point endWrite = steady_clock::now (); + perf->push_back (timing (startWrite, endWrite)); + } +} + +// +// memory allocation to hold all the pixels for a part, and a buffer that represents it +// +struct partBuffers +{ + vector> + scanlinePixelData; // pixel data for scanline input images - an array per channel + vector>> + tilePixelData; // pixel data for tiled input images - an array per level per channel + vector deepSampleCount; // per-pixel sample counts + vector> + deepSampleData; // actual sample data for deep scanline and deep single level deep tiled data + vector> + deepSamplePtrs; // pointers to deep sample data, a pointer per pixel per channel + FrameBuffer scanlineBuf; + vector tiledBuf; // + DeepFrameBuffer deepbuf; +}; + +struct partData +{ + partBuffers readBuf; + partBuffers rereadBuf; +}; + +void +initAndReadFile ( + MultiPartInputFile& in, + const vector
& outHeaders, + int part, + vector& parts, + fileMetrics& metrics, + bool reread) +{ + + for (size_t p = 0; p < parts.size (); ++p) + { + + int readPart = (part == -1 ? p : part); + string type = in.header (readPart).type (); + + if (type == SCANLINEIMAGE) + { + InputPart inpart (in, readPart); + metrics.stats[p].sizeData = initScanLine ( + parts[p].readBuf.scanlinePixelData, + parts[p].readBuf.scanlineBuf, + inpart, + outHeaders[p]); + if (reread) + { + initScanLine ( + parts[p].rereadBuf.scanlinePixelData, + parts[p].rereadBuf.scanlineBuf, + inpart, + outHeaders[p]); + } + readScanLine ( + inpart, + parts[p].readBuf.scanlineBuf, + metrics.stats[p].readPerf); + } + else if (type == TILEDIMAGE) + { + TiledInputPart inpart (in, readPart); + metrics.stats[p].sizeData = initTiled ( + parts[p].readBuf.tilePixelData, + parts[p].readBuf.tiledBuf, + inpart, + outHeaders[p]); + if (reread) + { + initTiled ( + parts[p].rereadBuf.tilePixelData, + parts[p].rereadBuf.tiledBuf, + inpart, + outHeaders[p]); + } + readTiled ( + inpart, parts[p].readBuf.tiledBuf, metrics.stats[p].readPerf); + } + else if (type == DEEPSCANLINE) + { + DeepScanLineInputPart inpart (in, readPart); + metrics.stats[p].sizeData = initAndReadDeepScanLine ( + parts[p].readBuf.deepSampleCount, + parts[p].readBuf.deepSampleData, + parts[p].readBuf.deepSamplePtrs, + parts[p].readBuf.deepbuf, + inpart, + nullptr, + outHeaders[p], + metrics.stats[p].countReadPerf, + metrics.stats[p].readPerf); + + if (reread) + { + metrics.stats[p].sizeData = initAndReadDeepScanLine ( + parts[p].rereadBuf.deepSampleCount, + parts[p].rereadBuf.deepSampleData, + parts[p].rereadBuf.deepSamplePtrs, + parts[p].rereadBuf.deepbuf, + inpart, + &parts[p].readBuf.deepSampleCount, + outHeaders[p], + metrics.stats[p].countReadPerf, + metrics.stats[p].readPerf); + } + } + else if (type == DEEPTILE) + { + DeepTiledInputPart inpart (in, readPart); + metrics.stats[p].sizeData = initAndReadDeepTiled ( + parts[p].readBuf.deepSampleCount, + parts[p].readBuf.deepSampleData, + parts[p].readBuf.deepSamplePtrs, + parts[p].readBuf.deepbuf, + inpart, + nullptr, + outHeaders[p], + metrics.stats[p].countReadPerf, + metrics.stats[p].readPerf); + if (reread) + { + initAndReadDeepTiled ( + parts[p].rereadBuf.deepSampleCount, + parts[p].rereadBuf.deepSampleData, + parts[p].rereadBuf.deepSamplePtrs, + parts[p].rereadBuf.deepbuf, + inpart, + &parts[p].readBuf.deepSampleCount, + outHeaders[p], + metrics.stats[p].countReadPerf, + metrics.stats[p].readPerf); + } + } + } +} - steady_clock::time_point startWrite = steady_clock::now(); - out.writeTiles (0, in.numXTiles (0) - 1, 0, in.numYTiles (0) - 1, 0, 0); - steady_clock::time_point endWrite = steady_clock::now(); +void +writeFile ( + MultiPartOutputFile& out, + vector& parts, + fileMetrics& metrics, + bool logPerformance) +{ + // + // Call initialization functions, Read image from source. + // + for (size_t p = 0; p < parts.size (); ++p) + { + string type = out.header (p).type (); + if (type == SCANLINEIMAGE) + { + OutputPart outpart (out, p); + writeScanLine ( + outpart, + parts[p].readBuf.scanlineBuf, + logPerformance ? &metrics.stats[p].writePerf : nullptr); + } + else if (type == TILEDIMAGE) + { + TiledOutputPart outpart (out, p); + writeTiled ( + outpart, + parts[p].readBuf.tiledBuf, + logPerformance ? &metrics.stats[p].writePerf : nullptr); + } + else if (type == DEEPSCANLINE) + { + DeepScanLineOutputPart outpart (out, p); + writeDeepScanLine ( + outpart, + parts[p].readBuf.deepbuf, + logPerformance ? &metrics.stats[p].writePerf : nullptr); + } + else if (type == DEEPTILE) + { + DeepTiledOutputPart outpart (out, p); + writeDeepTiled ( + outpart, + parts[p].readBuf.deepbuf, + logPerformance ? &metrics.stats[p].writePerf : nullptr); + } + } +} + +void +rereadFile ( + MultiPartInputFile& in, vector& parts, fileMetrics& metrics) +{ + // + // Call initialization functions, Read image from source. + // + for (size_t p = 0; p < parts.size (); ++p) + { + string type = in.header (p).type (); - cout << " \"count read time\": " << timing (startCountRead, endCountRead) - << ",\n"; - cout << " \"sample read time\": " - << timing (startSampleRead, endSampleRead) << ",\n"; - cout << " \"write time\": " << timing (startWrite, endWrite) << ",\n"; - cout << " \"pixel count\": " << numPixels << ",\n"; - cout << " \"raw size\": " - << totalSamples * bytesPerSample + numPixels * sizeof (int) << ",\n"; + if (type == SCANLINEIMAGE) + { + InputPart inpart (in, p); + readScanLine ( + inpart, + parts[p].rereadBuf.scanlineBuf, + metrics.stats[p].rereadPerf); + } + else if (type == TILEDIMAGE) + { + TiledInputPart inpart (in, p); + readTiled ( + inpart, + parts[p].rereadBuf.tiledBuf, + metrics.stats[p].rereadPerf); + } + else if (type == DEEPSCANLINE) + { + DeepScanLineInputPart inpart (in, p); + readDeepScanLine ( + inpart, + parts[p].rereadBuf.deepbuf, + metrics.stats[p].rereadPerf, + metrics.stats[p].countRereadPerf); + } + else if (type == DEEPTILE) + { + DeepTiledInputPart inpart (in, p); + readDeepTiled ( + inpart, + parts[p].rereadBuf.deepbuf, + metrics.stats[p].rereadPerf, + metrics.stats[p].countRereadPerf); + } + } } +// stream that doesn't write data, just logs file size +class DummyOStream : public OStream +{ + uint64_t streamptr = 0; + +public: + DummyOStream () : OStream ("") {} + + void write (const char c[], int n) override { streamptr += n; } + void seekp (uint64_t pos) override { streamptr = pos; } + uint64_t tellp () override { return streamptr; } +}; + +// +// write file to preallocated memory: must be initialized with enough space +// to store file +// +class MemOStream : public OStream +{ + + uint64_t streamptr = 0; + +public: + vector data; + MemOStream (uint64_t size) : OStream (""), data (size) {} + + void write (const char c[], int n) override + { + if (n + streamptr > data.size ()) + { + throw runtime_error ("attempt to write beyond preallocated memory"); + } + memcpy (data.data () + streamptr, c, n); + streamptr += n; + } + void seekp (uint64_t pos) override { streamptr = pos; } + uint64_t tellp () override { return streamptr; } +}; + +class MemIStream : public IStream +{ + uint64_t streamptr = 0; + const MemOStream& ostream; + +public: + MemIStream (const MemOStream& ostream) + : IStream (""), ostream (ostream) + {} + + bool isMemoryMapped () const override { return true; } + + char* readMemoryMapped (int n) override + { + if (n + streamptr > ostream.data.size ()) + { + throw runtime_error ("attempt to read past end of file"); + } + uint64_t oldStreamptr = streamptr; + streamptr += n; + return const_cast (ostream.data.data () + oldStreamptr); + } + + bool read (char c[], int n) override + { + int bytesToRead = + min (n, static_cast (ostream.data.size () - streamptr)); + memcpy (c, ostream.data.data () + streamptr, bytesToRead); + streamptr += n; + return bytesToRead < n; + } + + void seekg (uint64_t pos) override { streamptr = pos; } + + uint64_t tellg () override { return streamptr; } +}; + +// add each entry in input to the corresponding value in output +// if output has fewer entries than input, resize it to be the same size void +accumulate (vector& output, const vector& input) +{ + if (output.size () < input.size ()) { output.resize (input.size ()); } + for (size_t x = 0; x < input.size (); ++x) + { + output[x] += input[x]; + } +} + +string +modeName (PixelMode p) +{ + switch (p) + { + case PIXELMODE_ALL_FLOAT: return "float"; + case PIXELMODE_ALL_HALF: return "half"; + case PIXELMODE_MIXED_HALF_FLOAT: return "mixed"; + case PIXELMODE_ORIGINAL: return "original"; + } + throw runtime_error ("bad pixelmode"); +} + +fileMetrics exrmetrics ( - const char inFileName[], - const char outFileName[], - int part, + const char* inFileName, + const char* outFileName, + int part, OPENEXR_IMF_NAMESPACE::Compression compression, - float level, - int halfMode) + float level, + int passes, + bool write, + bool reread, + PixelMode pixelMode, + bool verbose) { + + if (verbose) + { + cerr << "read " << inFileName; + cerr << " as " << modeName (pixelMode) << "... "; + cerr.flush (); + } + MultiPartInputFile in (inFileName); - if (part >= in.parts ()) + if (part != -1 && part >= in.parts ()) { throw runtime_error ((string (inFileName) + " only contains " + to_string (in.parts ()) + " parts. Cannot copy part " + to_string (part)) .c_str ()); } - Header outHeader = in.header (part); + fileMetrics metrics; - if (compression < NUM_COMPRESSION_METHODS) + //write all parts if part==-1, otherwise write single part specified + vector
outHeaders (part == -1 ? in.parts () : 1); + if (part == -1) { - outHeader.compression () = compression; + for (int p = 0; p < in.parts (); ++p) + { + outHeaders[p] = in.header (p); + } } - else { compression = outHeader.compression (); } + else { outHeaders[0] = in.header (part); } + + bool compressionSet = false; - if (!isinf (level) && level >= -1) + for (int p = 0; p < in.parts (); ++p) { - switch (outHeader.compression ()) + if (compression < NUM_COMPRESSION_METHODS) { - case DWAA_COMPRESSION: - case DWAB_COMPRESSION: - outHeader.dwaCompressionLevel () = level; - break; - case ZIP_COMPRESSION: - case ZIPS_COMPRESSION: - outHeader.zipCompressionLevel () = level; - break; - // case ZSTD_COMPRESSION : - // outHeader.zstdCompressionLevel()=level; - // break; - default: - throw runtime_error ( - "-l option only works for DWAA/DWAB,ZIP/ZIPS or ZSTD compression"); + outHeaders[p].compression () = compression; } - } - if (halfMode > 0) - { - for (ChannelList::Iterator i = outHeader.channels ().begin (); - i != outHeader.channels ().end (); - ++i) + if (!isinf (level) && level >= -1) { - if (halfMode == 2 || !strcmp (i.name (), "R") || - !strcmp (i.name (), "G") || !strcmp (i.name (), "B") || - !strcmp (i.name (), "A")) + switch (outHeaders[p].compression ()) { - i.channel ().type = HALF; + case DWAA_COMPRESSION: + case DWAB_COMPRESSION: + outHeaders[p].dwaCompressionLevel () = level; + compressionSet = true; + break; + case ZIP_COMPRESSION: + case ZIPS_COMPRESSION: + outHeaders[p].zipCompressionLevel () = level; + compressionSet = true; + break; + // case ZSTD_COMPRESSION : + // outHeader.zstdCompressionLevel()=level; + // break; + default: break; } } - } - string inCompress, outCompress; - getCompressionNameFromId (in.header (part).compression (), inCompress); - getCompressionNameFromId (outHeader.compression (), outCompress); - cout << "{\n"; - cout << " \"input compression\": \"" << inCompress << "\",\n"; - cout << " \"output compression\": \"" << outCompress << "\",\n"; - if (compression == ZIP_COMPRESSION || compression == ZIPS_COMPRESSION) - { - cout << " \"zipCompressionLevel\": " - << outHeader.zipCompressionLevel () << ",\n"; + if (pixelMode != PIXELMODE_ORIGINAL) + { + for (ChannelList::Iterator i = outHeaders[p].channels ().begin (); + i != outHeaders[p].channels ().end (); + ++i) + { + // find channel suffix within full channel name (so 'R' in 'layer.R') + const char* name = i.name (); + const char* dot = strrchr (name, 'r'); + if (dot) { name = dot + 1; } + + if (pixelMode == PIXELMODE_ALL_HALF || + (pixelMode == PIXELMODE_MIXED_HALF_FLOAT && + (!strcmp (name, "R") || !strcmp (name, "G") || + !strcmp (name, "B") || !strcmp (name, "A")))) + { + i.channel ().type = HALF; + } + else if ( + pixelMode == PIXELMODE_ALL_FLOAT || + pixelMode == PIXELMODE_MIXED_HALF_FLOAT) + { + i.channel ().type = FLOAT; + } + } + } } - if (compression == DWAA_COMPRESSION || compression == DWAB_COMPRESSION) + // abort if level was set but no parts had a compression type with a level + if (!isinf (level) && level >= -1 && !compressionSet) { - cout << " \"dwaCompressionLevel\": " - << outHeader.dwaCompressionLevel () << ",\n"; + throw runtime_error ( + "-l option only works for DWAA/DWAB,ZIP/ZIPS or ZSTD compression"); } - std::string type = outHeader.type (); - cout << " \"part type\": \"" << type << "\",\n"; + vector parts (part == -1 ? in.parts () : 1); + metrics.stats.resize (parts.size ()); - if (type == SCANLINEIMAGE) - { - cout << " \"scanlines per chunk:\" : " - << getCompressionNumScanlines (compression) << ",\n"; - } + initAndReadFile (in, outHeaders, part, parts, metrics, reread); + if (write) { - MultiPartOutputFile out (outFileName, &outHeader, 1); - if (type == TILEDIMAGE) + // + // when NOT writing to file, write to preallocated block of data + // precompute the total datasize, so no memory reallocation is required + // + uint64_t fileSize = 0; + if (!outFileName) { - TiledInputPart inpart (in, part); - TiledOutputPart outpart (out, 0); - copyTiled (inpart, outpart); + + DummyOStream tmp; + MultiPartOutputFile out ( + tmp, outHeaders.data (), outHeaders.size ()); + writeFile (out, parts, metrics, false); + fileSize = tmp.tellp (); } - else if (type == SCANLINEIMAGE) + + // + // write to output; re-read output back to input + // + + if (verbose) { - InputPart inpart (in, part); - OutputPart outpart (out, 0); - copyScanLine (inpart, outpart); + cerr << " write "; + if (compression != NUM_COMPRESSION_METHODS) + { + string name; + getCompressionNameFromId (compression, name); + cerr << "compression " << name; + } + cerr << "... "; + cerr.flush (); } - else if (type == DEEPSCANLINE) + + for (int i = 0; i < passes; ++i) + { + if (verbose && passes > 1) + { + cerr << i << ' '; + cerr.flush (); + } + MemOStream ostream (fileSize); + MultiPartOutputFile* out; + if (outFileName) + { + out = new MultiPartOutputFile ( + outFileName, outHeaders.data (), outHeaders.size ()); + } + else + { + out = new MultiPartOutputFile ( + ostream, outHeaders.data (), outHeaders.size ()); + } + writeFile (*out, parts, metrics, true); + delete out; + + if (reread) + { + MemIStream istream (ostream); + MultiPartInputFile* in; + if (outFileName) { in = new MultiPartInputFile (outFileName); } + else { in = new MultiPartInputFile (istream); } + + rereadFile (*in, parts, metrics); + + delete in; + } + } + + struct stat instats, outstats; + stat (inFileName, &instats); + metrics.inputFileSize = instats.st_size; + if (outFileName) { - DeepScanLineInputPart inpart (in, part); - DeepScanLineOutputPart outpart (out, 0); - copyDeepScanLine (inpart, outpart); + stat (outFileName, &outstats); + metrics.outputFileSize = outstats.st_size; } - else if (type == DEEPTILE) + else { metrics.outputFileSize = fileSize; } + } + + // + // sum across all parts + // + + metrics.totalStats = metrics.stats[0]; + for (size_t i = 1; i < metrics.stats.size (); ++i) + { + accumulate (metrics.totalStats.readPerf, metrics.stats[i].readPerf); + accumulate ( + metrics.totalStats.countReadPerf, metrics.stats[i].countReadPerf); + accumulate (metrics.totalStats.writePerf, metrics.stats[i].writePerf); + accumulate (metrics.totalStats.rereadPerf, metrics.stats[i].rereadPerf); + accumulate ( + metrics.totalStats.countRereadPerf, + metrics.stats[i].countRereadPerf); + + metrics.totalStats.sizeData.pixelCount += + metrics.stats[i].sizeData.pixelCount; + metrics.totalStats.sizeData.channelCount += + metrics.stats[i].sizeData.channelCount; + metrics.totalStats.sizeData.rawSize += + metrics.stats[i].sizeData.rawSize; + metrics.totalStats.sizeData.tileCount += + metrics.stats[i].sizeData.tileCount; + + metrics.totalStats.sizeData.isDeep |= metrics.stats[i].sizeData.isDeep; + metrics.totalStats.sizeData.isTiled |= + metrics.stats[i].sizeData.isTiled; + + //check for mixed compression or part types within file + + if (metrics.stats[i].sizeData.compression != + metrics.totalStats.sizeData.compression) { - DeepTiledInputPart inpart (in, part); - DeepTiledOutputPart outpart (out, 0); - copyDeepTiled (inpart, outpart); + metrics.totalStats.sizeData.compression = NUM_COMPRESSION_METHODS; } - else + if (metrics.stats[i].sizeData.partType != + metrics.totalStats.sizeData.partType) { - throw runtime_error ( - (inFileName + string (" contains unknown part type ") + type) - .c_str ()); + metrics.totalStats.sizeData.partType = ""; } } - struct stat instats, outstats; - stat (inFileName, &instats); - stat (outFileName, &outstats); - cout << " \"input file size\": " << instats.st_size << ",\n"; - cout << " \"output file size\": " << outstats.st_size << "\n"; - cout << "}\n"; + + if (verbose) { cerr << endl; } + return metrics; } diff --git a/src/bin/exrmetrics/exrmetrics.h b/src/bin/exrmetrics/exrmetrics.h index fac98ee93..826e1c3fb 100644 --- a/src/bin/exrmetrics/exrmetrics.h +++ b/src/bin/exrmetrics/exrmetrics.h @@ -10,11 +10,68 @@ #include "ImfCompression.h" -void exrmetrics ( - const char inFileName[], - const char outFileName[], - int part, +#include "stdint.h" + +#include + +enum PixelMode +{ + PIXELMODE_ORIGINAL, + PIXELMODE_ALL_HALF, + PIXELMODE_ALL_FLOAT, + PIXELMODE_MIXED_HALF_FLOAT +}; +std::string modeName (PixelMode p); + +struct partSizeData +{ + uint64_t rawSize = + 0; // total size required to store just the pixel data, not including extra space + uint64_t pixelCount = + 0; // number of pixels in the image, including pixels mipmap levels for tiled images + uint64_t channelCount = 0; + uint64_t tileCount = 0; // for tiled images, the number of tiles + bool isDeep = false; + bool isTiled = false; + OPENEXR_IMF_NAMESPACE::Compression compression = + OPENEXR_IMF_NAMESPACE::NUM_COMPRESSION_METHODS; + std::string partType = ""; +}; + +struct partStats +{ + std::vector + countReadPerf; // for deep only, time reading the per-pixel sample count + std::vector readPerf; //time reading the pixel data + + std::vector writePerf; // time to write data (all part types) + + std::vector + countRereadPerf; // for deep only, time rereading the per-pixel sample count + std::vector + rereadPerf; // for deep, times reading the sample count, otherwise times reading the entire data + + partSizeData sizeData; +}; + +struct fileMetrics +{ + std::vector stats; + partStats totalStats; + uint64_t inputFileSize; + uint64_t outputFileSize; +}; + +fileMetrics exrmetrics ( + const char* inFileName, + const char* outFileName, + int part, OPENEXR_IMF_NAMESPACE::Compression compression, - float level, - int halfMode); + float level, + int passes, + bool write, + bool reread, + PixelMode pixelMode, + bool verbose); + #endif diff --git a/src/bin/exrmetrics/main.cpp b/src/bin/exrmetrics/main.cpp index b188b941f..47e616f59 100644 --- a/src/bin/exrmetrics/main.cpp +++ b/src/bin/exrmetrics/main.cpp @@ -10,8 +10,13 @@ #include "ImfMisc.h" #include "ImfThreading.h" #include "IlmThreadPool.h" +#include "ImfMultiPartInputFile.h" +#include "ImfVersion.h" +#include #include +#include +#include #include #include @@ -21,6 +26,12 @@ using std::cerr; using std::cout; using std::endl; +using std::list; +using std::max; +using std::min; +using std::sort; + +using std::numeric_limits; using std::ostream; using std::vector; using namespace OPENEXR_IMF_NAMESPACE; @@ -29,7 +40,8 @@ using namespace ILMTHREAD_NAMESPACE; void usageMessage (ostream& stream, const char* program_name, bool verbose = false) { - stream << "Usage: " << program_name << " [options] infile outfile" << endl; + stream << "Usage: " << program_name + << " [options] infile [infile2...] [-o outfile]" << endl; if (verbose) { @@ -41,45 +53,530 @@ usageMessage (ostream& stream, const char* program_name, bool verbose = false) "\n" "Options:\n" "\n" - " -p n part number to copy (only one part will be written to output file)\n" - " default is part 0\n" + " -o file file to write to. If no file specified, uses a memory buffer\n" + " note: file may be overwritten multiple times during tests\n" + " -p n part number to copy, or \"all\" for all parts\n" + " default is \"all\" \n" "\n" - " -m set to multi-threaded (system selected thread count)\n" - " -t n use n threads for processing files\n" - " default is single / no threads\n" + " -m set to multi-threaded (system selected thread count)\n" + " -t n use n threads for processing files\n" + " default is single / no threads\n" "\n" - " -l level set DWA or ZIP compression level\n" + " -l level set DWA or ZIP compression level\n" "\n" - " -z x sets the data compression method to x\n" + " -z,--compression list list of compression methods to test\n" " (" << compressionNames.c_str () - << ",\n" - " default retains original method)\n" + << ",orig,all\n" + " default orig: retains original method)\n" + + " --convert shorthand options for writing a new file with no metrics:\n" + " -p all --type orig --time none --type orig --no-size --passes 1\n" + " change pixel type or compression by specifying --type or -z after --convert\n" + " --bench shorthand options for robust performance benchmarking:\n" + " -p all --compression all --time write,reread --passes 10 --type half,float --no-size --csv\n" "\n" - " -16 rgba|all force 16 bit half float: either just RGBA, or all channels\n" - " default retains original type for all channels\n" + " -16 rgba|all [DEPRECATED] force 16 bit half float: either just RGBA, or all channels\n" + " Use --type half or --type mixed instead\n" + " --pixelmode list list of pixel types to use (float,half,mixed,orig)\n" + " mixed uses half for RGBA, float for others. Default is 'orig'\n" + " --time list comma separated list of operations to report timing for.\n" + " operations can be any of read,write,reread (use --time none for no timing)\n" + " --no-size don't output size data\n" + " --json print output as JSON dictionary (Default mode)\n" + " --csv print output in csv mode. If passes>1, show median timing\n" + " default is JSON mode\n" + " --passes num write and re-read file num times (default 1)\n" "\n" - " -h, --help print this message\n" + " -h, --help print this message\n" + " -v output progress messages\n" "\n" - " --version print version information\n" + " --version print version information\n" "\n"; } } +struct options +{ + enum + { + TIME_NONE = 0, + TIME_READ = 1, + TIME_WRITE = 2, + TIME_REREAD = 4 + }; + + const char* outFile = nullptr; + std::vector inFiles; + int part = -1; + int threads = 0; + float level = INFINITY; + int passes = 1; + int timing = TIME_READ | TIME_REREAD | TIME_WRITE; + bool outputSizeData = true; + bool verbose = false; + bool csv = false; + std::vector pixelModes; + std::vector compressions; + + int parse (int argc, char* argv[]); +}; + +struct runData +{ + const char* file; + PixelMode mode; + Compression compression; + fileMetrics metrics; +}; + +double +median (const vector& perf) +{ + if (perf.size () == 1) { return perf[0]; } + vector d = perf; + sort (d.begin (), d.end ()); + return (d[d.size () / 2] + d[(d.size () - 1) / 2]) / 2.0; +} + +void +printTiming (const vector& perf, ostream& out, bool raw, bool stats) +{ + + if (perf.size () == 0) { return; } + + if (perf.size () == 1) + { + out << perf[0]; + return; + } + + out << '{'; + if (raw) { out << "\"values\": [ "; } + double n = perf.size (); + double k = perf[0]; + double sum = 0.; + double x = 0.; + double x2 = 0.; + double minimum = numeric_limits::max (); + double maximum = -minimum; + + bool first = true; + for (double i: perf) + { + + if (raw) + { + if (!first) { out << " , "; } + first = false; + out << i; + } + sum += i; + x += i - k; + x2 += (i - k) * (i - k); + maximum = max (maximum, i); + minimum = min (minimum, i); + } + if (raw) { out << " ] , "; } + if (stats) + { + out << "\"min\": " << minimum << ", \"max\": " << maximum + << ", \"mean\": " << sum / n << ", \"median\": " << median (perf) + << ", \"std dev\": "; + out << sqrt ((x2 - x * x / n) / n); + } + out << "}"; +} + +void +printPartStats ( + ostream& out, + const partStats& data, + const string indent, + int timing, + bool raw, + bool stats) +{ + bool output = false; + if (timing & options::TIME_READ) + { + if (data.sizeData.isDeep) + { + out << indent << "\"count read time\": "; + printTiming (data.countReadPerf, out, raw, stats); + output = true; + } + if (output) { out << ",\n"; } + out << indent << "\"read time\": "; + printTiming (data.readPerf, out, raw, stats); + output = true; + } + + if (timing & options::TIME_WRITE) + { + if (output) { out << ",\n"; } + output = true; + out << indent << "\"write time\": "; + printTiming (data.writePerf, out, raw, stats); + } + + if (timing & options::TIME_REREAD) + { + if (data.sizeData.isDeep) + { + if (output) { out << ",\n"; } + output = true; + out << indent << "\"count re-read time\": "; + printTiming (data.countRereadPerf, out, raw, stats); + } + if (output) { out << ",\n"; } + output = true; + out << indent << "\"re-read time\": "; + printTiming (data.rereadPerf, out, raw, stats); + } + if (output) { out << '\n'; } +} + +void +jsonStats ( + ostream& out, + list& data, + bool outputSizeData, + int timing, + bool raw, + bool stats) +{ + + static const char* lastFileName = nullptr; + out << '[' << endl; + bool firstEntryForFile = true; + for (runData run: data) + { + if (run.file != lastFileName) + { + if (lastFileName != nullptr) + { + out << "\n ]\n"; + out << " },\n"; + } + out << " {\n" + << " \"file\": \"" << run.file << "\",\n"; + lastFileName = run.file; + if (outputSizeData) + { + out << " \"input file size\": " << run.metrics.inputFileSize + << ",\n"; + out << " \"pixels\": " + << run.metrics.totalStats.sizeData.pixelCount << ",\n"; + out << " \"channels\": " + << run.metrics.totalStats.sizeData.channelCount << ",\n"; + out << " \"total raw size\": " + << run.metrics.totalStats.sizeData.rawSize << ",\n"; + string compName; + if (run.metrics.totalStats.sizeData.compression == + NUM_COMPRESSION_METHODS) + { + compName = "mixed"; + } + else + { + getCompressionNameFromId ( + run.metrics.totalStats.sizeData.compression, compName); + } + out << " \"compression\": \"" << compName << "\",\n"; + + out << " \"part type\": \""; + if (run.metrics.totalStats.sizeData.partType == "") + { + out << "mixed"; + } + else { out << run.metrics.totalStats.sizeData.partType; } + out << "\",\n"; + + if (run.metrics.totalStats.sizeData.isTiled) + { + out << " \"tile count\": " + << run.metrics.totalStats.sizeData.tileCount << ",\n"; + } + + if (run.metrics.stats.size () > 0) + { + out << " \"parts\":\n"; + out << " [\n"; + + for (size_t part = 0; part < run.metrics.stats.size (); + ++part) + { + out << " {\n"; + out << " \"part\": " << part << ",\n"; + out << " \"pixels\": " + << run.metrics.stats[part].sizeData.pixelCount + << ",\n"; + out << " \"channels\": " + << run.metrics.stats[part].sizeData.channelCount + << ",\n"; + string compName; + getCompressionNameFromId ( + run.metrics.stats[part].sizeData.compression, + compName); + out << " \"compression\": \"" << compName + << "\",\n"; + out << " \"part type\": \"" + << run.metrics.stats[part].sizeData.partType + << "\",\n"; + if (run.metrics.stats[part].sizeData.isTiled) + { + out << " \"tile count\": " + << run.metrics.stats[part].sizeData.tileCount + << ",\n"; + } + out << " \"total raw size\": " + << run.metrics.stats[part].sizeData.rawSize << "\n"; + out << " }"; + if (part < run.metrics.stats.size () - 1) + { + out << ','; + } + out << "\n"; + } + out << " ],\n"; + } + out << " \"metrics\":\n"; + out << " ["; + firstEntryForFile = true; + } + } + + string compName; + if (run.compression == NUM_COMPRESSION_METHODS) + { + compName = "original"; + } + else { getCompressionNameFromId (run.compression, compName); } + + if (!firstEntryForFile) { out << ','; } + out << '\n'; + out << " {\n"; + out << " \"compression\": \"" << compName << "\",\n"; + out << " \"pixel mode\": \"" << modeName (run.mode) << "\",\n"; + + if (outputSizeData) + { + out << " \"output size\": " << run.metrics.outputFileSize + << ",\n"; + } + + printPartStats ( + out, run.metrics.totalStats, " ", timing, raw, stats); + if (run.metrics.stats.size () > 1) + { + out << " \"parts\":\n"; + out << " [\n"; + //first print total statistics, then print all part data, unless there's only one part + for (size_t part = 0; part < run.metrics.stats.size (); ++part) + { + out << " {\n"; + out << " \"part\": " << part << ",\n"; + printPartStats ( + out, + run.metrics.stats[part], + " ", + timing, + raw, + stats); + out << " }"; + if (part < run.metrics.stats.size () - 1) { out << ','; } + out << endl; + } + out << " ]\n"; + } + out << " }"; + firstEntryForFile = false; + } + + if (lastFileName) + { + out << "\n ]\n"; + out << " }\n"; + } + out << "]\n"; +} + +void +csvStats (ostream& out, list& data, bool outputSizeData, int timing) +{ + out << "file name"; + if (outputSizeData) + { + out << ",input size,pixel count,channel count,tile count,raw size"; + } + out << ",compression,pixel mode"; + if (outputSizeData) { out << ",output size"; } + if (timing & options::TIME_READ) + { + out << ",count read time"; + out << ",read time"; + } + if (timing & options::TIME_WRITE) { out << ",write time"; } + if (timing & options::TIME_REREAD) + { + out << ",count reread time"; + out << ",reread time"; + } + cout << "\n"; + for (runData run: data) + { + out << run.file; + if (outputSizeData) + { + out << ',' << run.metrics.inputFileSize << ',' + << run.metrics.totalStats.sizeData.pixelCount << ',' + << run.metrics.totalStats.sizeData.channelCount; + if (run.metrics.totalStats.sizeData.isTiled) + { + out << ',' << run.metrics.totalStats.sizeData.tileCount; + } + else { out << ",---"; } + out << ',' << run.metrics.totalStats.sizeData.rawSize; + } + string compName; + if (run.compression == NUM_COMPRESSION_METHODS) + { + compName = "original"; + } + else { getCompressionNameFromId (run.compression, compName); } + out << ',' << compName << ',' << modeName (run.mode); + + if (outputSizeData) { out << ',' << run.metrics.outputFileSize; } + if (timing & options::TIME_READ) + { + if (run.metrics.totalStats.sizeData.isDeep) + { + out << ',' << median (run.metrics.totalStats.countReadPerf); + } + else { out << ",---"; } + out << ',' << median (run.metrics.totalStats.readPerf); + } + if (timing & options::TIME_WRITE) + { + out << ',' << median (run.metrics.totalStats.writePerf); + } + if (timing & options::TIME_REREAD) + { + if (run.metrics.totalStats.sizeData.isDeep) + { + out << ',' << median (run.metrics.totalStats.countRereadPerf); + } + else { out << ",---"; } + out << ',' << median (run.metrics.totalStats.rereadPerf); + } + out << "\n"; + } +} + int main (int argc, char** argv) { - const char* outFile = nullptr; - const char* inFile = nullptr; - int part = 0; - int threads = 0; - float level = INFINITY; - int halfMode = 0; // 0 - leave alone, 1 - just RGBA, 2 - everything - Compression compression = Compression::NUM_COMPRESSION_METHODS; + options opts; + + if (opts.parse (argc, argv)) { return 1; } + + list data; + try + { + if (opts.threads < 0) + setGlobalThreadCount (ThreadPool::estimateThreadCountForFileIO ()); + else + setGlobalThreadCount (opts.threads); + + for (const char* inFile: opts.inFiles) + { + bool hasDeep = false; + + // + // unless using original compression method, check whether file is deep + // to skip incompatible compression methods + // + if (opts.compressions.size () > 1 && + opts.compressions[0] != NUM_COMPRESSION_METHODS) + { + MultiPartInputFile in (inFile); + hasDeep = isNonImage (in.version ()); + } + for (Compression compression: opts.compressions) + { + if (!hasDeep || compression == NUM_COMPRESSION_METHODS || + isValidDeepCompression (compression)) + { + for (PixelMode mode: opts.pixelModes) + { + runData d; + d.file = inFile; + d.compression = compression; + d.mode = mode; + d.metrics = exrmetrics ( + inFile, + opts.outFile, + opts.part, + compression, + opts.level, + opts.passes, + opts.outFile || opts.outputSizeData || + opts.timing & options::TIME_WRITE, + opts.timing & options::TIME_REREAD, + mode, + opts.verbose); + data.push_back (d); + } + } + } + } + } + catch (std::exception& what) + { + cerr << "error from exrmetrics: " << what.what () << endl; + return 1; + } + + if (opts.timing || opts.outputSizeData) + { + + if (opts.csv) + { + csvStats (cout, data, opts.outputSizeData, opts.timing); + } + else + { + jsonStats ( + cout, data, opts.outputSizeData, opts.timing, true, true); + } + } + + return 0; +} + +std::list +split (const char* str, char splitChar) +{ + std::istringstream stream (str); + std::list items; + std::string token; + while (std::getline (stream, token, splitChar)) + { + items.push_back (token); + } + return items; +} + +int +options::parse (int argc, char* argv[]) +{ int i = 1; + timing = TIME_READ | TIME_WRITE | TIME_REREAD; + if (argc == 1) { usageMessage (cerr, "exrmetrics", true); @@ -122,26 +619,108 @@ main (int argc, char** argv) threads = atoi (argv[i + 1]); if (threads < 0) { - cerr << "bad thread count " << argv[i + 1] << " specified to -t option\n"; + cerr << "bad thread count " << argv[i + 1] + << " specified to -t option\n"; return 1; } i += 2; } - else if (!strcmp (argv[i], "-z")) + else if (!strcmp (argv[i], "--bench")) + { + compressions.clear (); + for (int c = 0; c < NUM_COMPRESSION_METHODS; ++c) + { + compressions.push_back (Compression (c)); + } + passes = 10; + outputSizeData = false; + csv = true; + part = -1; + timing = TIME_WRITE | TIME_REREAD; + pixelModes.resize (2); + pixelModes[0] = PIXELMODE_ALL_HALF; + pixelModes[1] = PIXELMODE_ALL_FLOAT; + i += 1; + } + else if (!strcmp (argv[i], "--convert")) + { + pixelModes.resize (1); + pixelModes[0] = PIXELMODE_ORIGINAL; + passes = 1; + outputSizeData = false; + timing = 0; + part = -1; + i += 1; + } + else if (!strcmp (argv[i], "--passes")) { if (i > argc - 2) { - cerr << "Missing compression value with -z option\n"; + cerr << "Missing pass count value with --passes option\n"; return 1; } - - getCompressionIdFromName (argv[i + 1], compression); - if (compression == Compression::NUM_COMPRESSION_METHODS) + passes = atoi (argv[i + 1]); + if (passes < 0) + { + cerr << "bad value for passes " << argv[i + 1] + << " specified to --passes option\n"; + return 1; + } + i += 2; + } + else if (!strcmp (argv[i], "-v")) + { + verbose = true; + i += 1; + } + else if (!strcmp (argv[i], "--csv")) + { + csv = true; + i += 1; + } + else if (!strcmp (argv[i], "--json")) + { + csv = false; + i += 1; + } + else if (!strcmp (argv[i], "-z") || !strcmp (argv[i], "--compression")) + { + compressions.clear (); + if (i > argc - 2) { - cerr << "unknown compression type " << argv[i + 1] << endl; + cerr << "Missing compression value with " << argv[i] + << " option\n"; return 1; } + std::list items = split (argv[i + 1], ','); + + for (string i: items) + { + + if (i == "orig") + { + compressions.push_back (NUM_COMPRESSION_METHODS); + } + else if (i == "all") + { + for (int c = 0; c < NUM_COMPRESSION_METHODS; ++c) + { + compressions.push_back (Compression (c)); + } + } + else + { + Compression compression; + getCompressionIdFromName (i, compression); + if (compression == Compression::NUM_COMPRESSION_METHODS) + { + cerr << "unknown compression type " << i << endl; + return 1; + } + compressions.push_back (compression); + } + } i += 2; } else if (!strcmp (argv[i], "-p")) @@ -151,11 +730,15 @@ main (int argc, char** argv) cerr << "Missing part number with -p option\n"; return 1; } - part = atoi (argv[i + 1]); - if (part < 0) + if (!strcmp (argv[i + 1], "all")) { part = -1; } + else { - cerr << "bad part " << part << " specified to -p option\n"; - return 1; + part = atoi (argv[i + 1]); + if (part < -1) + { + cerr << "bad part " << part << " specified to -p option\n"; + return 1; + } } i += 2; @@ -176,6 +759,22 @@ main (int argc, char** argv) i += 2; } + else if (!strcmp (argv[i], "-o")) + { + if (i > argc - 2) + { + cerr << "Missing filename specified with -o\n"; + return 1; + } + if (outFile) + { + cerr << "-o output filename can only be specified once\n"; + return 1; + } + outFile = argv[i + 1]; + i += 2; + } + // deprecated flag for backwards compatibility else if (!strcmp (argv[i], "-16")) { if (i > argc - 2) @@ -183,8 +782,14 @@ main (int argc, char** argv) cerr << "Missing mode with -16 option\n"; return 1; } - if (!strcmp (argv[i + 1], "all")) { halfMode = 2; } - else if (!strcmp (argv[i + 1], "rgba")) { halfMode = 1; } + if (!strcmp (argv[i + 1], "all")) + { + pixelModes.push_back (PIXELMODE_ALL_HALF); + } + else if (!strcmp (argv[i + 1], "rgba")) + { + pixelModes.push_back (PIXELMODE_MIXED_HALF_FLOAT); + } else { cerr << " bad mode for -16 option: must be 'all' or 'rgba'\n"; @@ -192,43 +797,110 @@ main (int argc, char** argv) } i += 2; } - else if (!inFile) + else if (!strcmp (argv[i], "--pixelmode")) { - inFile = argv[i]; - i += 1; + if (i > argc - 2) + { + cerr << "Missing type list with with --pixelmode option\n"; + return 1; + } + std::list items = split (argv[i + 1], ','); + pixelModes.clear (); + for (string i: items) + { + if (i == "half") { pixelModes.push_back (PIXELMODE_ALL_HALF); } + else if (i == "float") + { + pixelModes.push_back (PIXELMODE_ALL_FLOAT); + } + else if (i == "rgba" || i == "mixed") + { + pixelModes.push_back (PIXELMODE_MIXED_HALF_FLOAT); + } + else if (i == "orig") + { + pixelModes.push_back (PIXELMODE_ORIGINAL); + } + else + { + cerr + << "bad pixel type " << i + << " for --pixelmode: must be half,float,rgba,mixed or orig\n"; + return 1; + } + } + i += 2; } - else if (!outFile) + else if (!strcmp (argv[i], "--time")) { - outFile = argv[i]; + if (i > argc - 2) + { + cerr << "Missing value list with --time option\n"; + return 1; + } + + timing = TIME_NONE; + if (strcmp (argv[i + 1], "none")) + { + std::list items = split (argv[i + 1], ','); + for (string i: items) + { + if (i == "read") { timing |= TIME_READ; } + else if (i == "reread") { timing |= TIME_REREAD; } + else if (i == "write") { timing |= TIME_WRITE; } + else + { + cerr + << "bad value in timing list. Options are read,write,reread\n"; + return 1; + } + } + } + i += 2; + } + else if (!strcmp (argv[i], "--no-size")) + { + outputSizeData = false; i += 1; } + else if (!strcmp (argv[i], "-i")) + { + if (i > argc - 2) + { + cerr << "Missing filename with -i option\n"; + return 1; + } + inFiles.push_back (argv[i + 1]); + i += 2; + } else { - cerr << "unknown argument or extra filename specified\n"; - usageMessage (cerr, "exrmetrics", false); - return 1; + inFiles.push_back (argv[i]); + i += 1; } } - if (!inFile || !outFile) + if (inFiles.size () == 0) { - cerr << "Missing input or output file\n"; + cerr << "Missing input file\n"; usageMessage (cerr, "exrmetrics", false); return 1; } - try + if (!outputSizeData && !timing && !outFile) { - if (threads < 0) - setGlobalThreadCount (ThreadPool::estimateThreadCountForFileIO ()); - else - setGlobalThreadCount (threads); - - exrmetrics (inFile, outFile, part, compression, level, halfMode); + cerr + << "Nothing to do: no output file specified, and all performance/size data disabled"; + cerr << "Use -o to specify output image filename\n"; + return 1; } - catch (std::exception& what) + + // default options if none specified + if (pixelModes.size () == 0) { pixelModes.push_back (PIXELMODE_ORIGINAL); } + + if (compressions.size () == 0) { - cerr << "error from exrmetrics: " << what.what () << endl; - return 1; + compressions.push_back (NUM_COMPRESSION_METHODS); } + return 0; } diff --git a/src/test/bin/test_exrmetrics.py b/src/test/bin/test_exrmetrics.py index d6f5b14a0..45b9f6080 100644 --- a/src/test/bin/test_exrmetrics.py +++ b/src/test/bin/test_exrmetrics.py @@ -48,15 +48,16 @@ def cleanup(): # test missing arguments, using just the -option but no value -for a in ["-p","-l","-16","-z"]: +for a in ["-p","-l","-16","-z","-t","-i","--passes","-o","--pixelmode","--time"]: result = run ([exrmetrics, a], stdout=PIPE, stderr=PIPE, universal_newlines=True) print(" ".join(result.args)) print(result.stderr) assert(result.returncode != 0), "\n"+result.stderr + assert("Missing" in result.stderr),"expected 'Missing argument' error" command = [exrmetrics] image = f"{image_dir}/TestImages/GrayRampsHorizontal.exr" -command += [image, outimage] +command += ["-i",image, "--passes","2","-o",outimage] result = run (command, stdout=PIPE, stderr=PIPE, universal_newlines=True) print(" ".join(result.args)) @@ -68,7 +69,8 @@ def cleanup(): # confirm data is valid JSON (will not be true if filename contains quotes) data = json.loads(result.stdout) -for x in ['write time','output file size','input file size']: - assert(x in data),"\n Missing field "+x +assert(len(data)==1),"\n Unexpected list size in JSON object" +for x in ['file','pixels','compression','part type','total raw size']: + assert(x in data[0]),"\n Missing field "+x print("success")