Every id must be a device candidate that fits one lane on its own. Plain files skip nvCOMP: they lead the lane's slots and reach them as read. Files with extra member signatures reach finish without device inflation.
854 {
855 auto&
lane = lanes_[lane_index];
857 lane.decoded =
false;
860
862 auto table = host_table(
lane);
863
864
865 uint64_t plain_at = 0;
866 for (size_t file = 0; file < files; ++file) {
867 auto const& probe = probes[
ids[file]];
868 if (probe.plain) plain_at += probe.host_bytes();
869 }
870 uint64_t compressed_at = plain_at, slot_at = 0;
871 for (size_t file = 0; file < files; ++file) {
872 auto const& probe = probes[
ids[file]];
873 table.compressed_bytes[file] = probe.compressed;
874 if (probe.plain) {
875 table.compressed_offsets[file] = slot_at + 1;
876 slot_at += probe.host_bytes();
877 } else {
878 table.compressed_offsets[file] = compressed_at;
879 compressed_at += probe.compressed;
880 }
881 }
882 std::atomic<bool> failed{false};
883 readers_.
for_each(files, [&](
size_t file) {
884 auto const*
const path = paths[
ids[file]].c_str();
885 auto*
const out =
lane.pinned->data() + table.compressed_offsets[file];
886 auto const bytes = table.compressed_bytes[file];
887 int const fd = ::open(path, O_RDONLY);
888
889 bool const read = fd != -1 && read_whole_file(fd, out, bytes, direct_reads);
890 auto const done = read ? bytes : 0;
891 if (!read) failed = true;
892
893
894
895 auto const& probe = probes[
ids[file]];
896 lane.possible_members[file] =
897 !probe.plain && !probe.bgzf && done > 10 && has_gzip_signature(out + 10, done - 10);
898 });
901 lane.fallback.clear();
902 lane.member_first.clear();
903 engine_params_.clear();
904 slot_at = 0;
905 uint32_t members = 0;
906 auto const accept = [&](size_t file) {
907 table.slots[
lane.files.size()] = slot_at;
908 lane.member_first.push_back(members);
909 slot_at += probes[
ids[file]].isize + 1;
910 lane.files.push_back(
ids[file]);
911 };
912 for (size_t file = 0; file < files; ++file) {
913 if (probes[
ids[file]].plain) accept(file);
914 }
915 auto const plain =
lane.files.size();
916 for (size_t file = 0; file < files; ++file) {
917 if (!probes[
ids[file]].plain &&
lane.possible_members[file]) {
918 lane.fallback.push_back(
ids[file]);
919 }
920 }
921
922
923 auto const add_member = [&](unsigned char const* deflate,
924 uint64_t deflate_bytes,
925 uint32_t size,
926 char* out) {
927 auto*
const source =
lane.payload->data() +
928 (
reinterpret_cast<char const*
>(deflate) -
lane.pinned->data());
929 table.member_sources[members] = source;
930 table.member_source_bytes[members] = deflate_bytes;
931 table.member_outputs[members] = out;
932 table.member_output_bytes[members] = size;
933 if (hardware_limit_ != 0 && size <= hardware_limit_) {
934 CUmemDecompressParams params{};
935 params.src = source;
936 params.srcNumBytes = deflate_bytes;
937 params.dst = out;
938 params.dstNumBytes = size;
939
940 params.dstActBytes =
941 reinterpret_cast<cuuint32_t*
>(device_tables(
lane).member_inflated + members);
942 params.algo = CU_MEM_DECOMPRESS_ALGORITHM_DEFLATE;
943 engine_params_.push_back(params);
944 }
945 ++members;
946 };
947
948
949
950
951 auto const gzip_pass = [&](bool engine) {
952 for (size_t file = 0; file < files; ++file) {
953 auto const& probe = probes[
ids[file]];
954 if (probe.plain ||
lane.possible_members[file])
continue;
955 bool const fits =
956 hardware_limit_ != 0 && (probe.bgzf || probe.isize <= hardware_limit_);
957 if (fits != engine) continue;
958 auto const* const bytes = reinterpret_cast<unsigned char const*>(
959 lane.pinned->data() + table.compressed_offsets[file]
960 );
961 auto* out =
lane.slots->data() + slot_at + 1;
962 if (probe.bgzf) {
963
964 auto const blocks =
walk_bgzf(bytes, probe.compressed, [](
auto...) {});
965 if (!blocks || blocks->first != probe.members ||
966 blocks->second != probe.isize) {
967 lane.fallback.push_back(
ids[file]);
968 continue;
969 }
970 accept(file);
972 bytes, probe.compressed, [&](uint64_t at, uint64_t length, uint32_t size) {
973 add_member(bytes + at, length, size, out);
974 out += size;
975 }
976 );
977 continue;
978 }
980 reinterpret_cast<char const*>(bytes), probe.compressed
981 );
982 if (deflate_at == 0) {
983 lane.fallback.push_back(
ids[file]);
984 continue;
985 }
986 accept(file);
987 add_member(bytes + deflate_at, probe.compressed - deflate_at - 8, probe.isize, out);
988 }
989 };
990 gzip_pass(true);
991 auto const engine_members = members;
992 gzip_pass(false);
993 lane.member_first.push_back(members);
994 table.slots[
lane.files.size()] = slot_at;
995 lane.slot_bytes = slot_at;
996 if (
lane.files.empty())
return files;
997
998 auto const s = cuda::stream_ref{*
lane.stream};
999 auto* const stream = s.get();
1000 auto const device_table = device_tables(
lane);
1001
1003
1004 if (plain_at != 0) {
1006 cuda::copy_bytes(
1007 s,
1008 cuda::std::span{
lane.pinned->data(), plain_at},
1009 device_span<char>{
lane.slots->data(), plain_at}
1010 )
1011 );
1012 }
1013 if (compressed_at != plain_at) {
1015 cuda::copy_bytes(
1016 s,
1017 cuda::std::span{
lane.pinned->data() + plain_at, compressed_at - plain_at},
1018 device_span<char>{
lane.payload->data() + plain_at, compressed_at - plain_at}
1019 )
1020 );
1021 }
1023 cuda::copy_bytes(
1024 s,
1025 cuda::std::span{
lane.host->data(), inputs_bytes()},
1026 device_span<char>{
lane.tables->data(), inputs_bytes()}
1027 )
1028 );
1030 auto const count =
static_cast<uint32_t
>(
lane.files.size());
1031 device_gzip::slot_init_kernel<<<(
count + 255) / 256, 256, 0, stream>>>(
1033 device_table.slots,
1035 device_table.first_header,
1036 device_table.header_count
1037 );
1039 if (engine_members != 0) {
1040
1041
1042
1043
1044 static_assert(nvcompSuccess == 0);
1045 static_assert(std::endian::native == std::endian::little);
1047 device_table.member_inflated, 0, engine_members * sizeof(uint64_t), stream
1048 ));
1050 device_table.member_statuses, 0, engine_members * sizeof(nvcompStatus_t), stream
1051 ));
1052 size_t error_index = 0;
1053 auto const submitted = batch_decompress_(
1054 engine_params_.data(), engine_params_.size(), 0, &error_index, stream
1055 );
1056 if (submitted != CUDA_SUCCESS) {
1059 "hardware decompression launch failed: " + std::to_string(submitted)
1060 )
1061 );
1062 }
1063 }
1064 if (engine_members != members) {
1065 auto options = nvcompBatchedDeflateDecompressDefaultOpts;
1066 options.backend = NVCOMP_DECOMPRESS_BACKEND_CUDA;
1067 auto const first = engine_members;
1068 auto const inflated = nvcompBatchedDeflateDecompressAsync(
1069 reinterpret_cast<void const* const*>(device_table.member_sources + first),
1070 device_table.member_source_bytes + first,
1071 device_table.member_output_bytes + first,
1072 device_table.member_inflated + first,
1073 members - first,
1075 nvcomp_temp_bytes_,
1076 reinterpret_cast<void* const*>(device_table.member_outputs + first),
1077 options,
1078 device_table.member_statuses + first,
1079 stream
1080 );
1081 if (inflated != nvcompSuccess) {
1083 Error::resource(
"nvCOMP DEFLATE launch failed: " + std::to_string(inflated))
1084 );
1085 }
1086 }
1087 if (
auto const gzip =
count -
static_cast<uint32_t
>(plain); gzip != 0) {
1088 device_gzip::slot_crc_kernel<<<gzip, device_gzip::block_size, 0, stream>>>(
1089 lane.slots->data(), device_table.slots + plain, device_table.crcs + plain
1090 );
1092 }
1094 return files;
1095 }
size_t batch_files(std::span< size_t const > ids, std::span< gzip_file_probe const > probes) const noexcept
How many ids from the front of ids one lane holds.
void for_each(size_t count, Work &&work)
Runs work(index) for every index below count; one call at a time.
#define CUDDL_TRY(expr)
Propagates a cuddl::Result failure from the enclosing function (GNU statement expression).
size_t deflate_offset(char const *bytes, size_t size)
Where the DEFLATE stream starts in the size bytes of a gzip candidate (one member,...
std::optional< std::pair< uint32_t, uint64_t > > walk_bgzf(unsigned char const *bytes, uint64_t size, Block &&block)
Calls block(deflate_at, deflate_bytes, inflated) for each non-empty block of the BGZF file in bytes.
Result< void > cuda_try(cudaError_t error, std::source_location location=std::source_location::current())
Checks a CUDA runtime call and returns an error on failure.
static Error invalid_argument(std::string message)