cuddl::reference_index_file Class Reference

Index-file persistence and the validation boundary for reference_index. More...

Static Public Member Functions
template<uint32_t K, size_t BucketCount, typename Layout >
static Result< void >	save (reference_index< K, BucketCount, Layout > const &index, reference_database< K, BucketCount, Layout > const &source, std::filesystem::path const &path, cuda::stream_ref stream)
	Atomically save an already built or loaded index.
template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>
static Result< reference_index< K, BucketCount, Layout > >	load (std::filesystem::path const &path, reference_database< K, BucketCount, Layout > const &database, cuda::stream_ref stream)
	Validate the file and its database binding once, then restore a ready-to-search index.

Detailed Description

Index-file persistence and the validation boundary for reference_index.

Definition at line 74 of file reference_index_file.cuh.

Member Function Documentation

load()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static Result< reference_index< K, BucketCount, Layout > > cuddl::reference_index_file::load ( std::filesystem::path const &  path, reference_database< K, BucketCount, Layout > const &  database, cuda::stream_ref  stream  )

inlinestatic

Validate the file and its database binding once, then restore a ready-to-search index.

Definition at line 144 of file reference_index_file.cuh.

      {
        detail::device_scope const scope{stream};
        auto source = CUDDL_TRY(snapshot(database, stream));
        auto decoded = CUDDL_TRY(read(path, std::move(source)));
        auto index = CUDDL_CUDA_TRY((reference_index<K, BucketCount, Layout>(database, stream)));
        index.index_offsets_ = CUDDL_CUDA_TRY(
            cuda::make_device_buffer<uint32_t>(stream, stream.device(), decoded.offsets_)
        );
        index.index_keys_ = CUDDL_CUDA_TRY(
            cuda::make_device_buffer<uint16_t>(stream, stream.device(), decoded.keys_)
        );
        index.index_posting_capacity_ = static_cast<size_t>(detail::indexed_posting_count(
            database.reference_count(), database.metadata().compatibility
        ));
        index.index_postings_ = CUDDL_CUDA_TRY(
            cuda::make_device_buffer<uint32_t>(
                stream, stream.device(), index.index_posting_capacity_, cuda::no_init
            )
        );
        if (!decoded.postings_.empty()) {
            CUDDL_CUDA_TRY(
                cuda::copy_bytes(
                    stream,
                    cuda::std::span{decoded.postings_.data(), decoded.postings_.size()},
                    cuda::std::span{index.index_postings_.data(), decoded.postings_.size()}
                )
            );
        }
        // The driver may still read the pageable host sources after the copies return.
        CUDDL_CUDA_TRY(stream.sync());
        CUDDL_TRY(index.build_key_directory(
            database.reference_count(),
            database.metadata().compatibility.indexed_bucket_count,
            stream
        ));
        index.indexed_ = true;
        index.pair_work_ = decoded.pair_work_;
        {
            auto const reference_count = database.reference_count();
            auto const indexed_buckets = database.metadata().compatibility.indexed_bucket_count;
            index.pair_fraction_ = reference_count == 0U || indexed_buckets == 0U
                                       ? 0.0
                                       : static_cast<double>(index.pair_work_) /
                                             (static_cast<double>(reference_count) *
                                              reference_count * indexed_buckets);
        }
        CUDDL_TRY(index.build_posting_bitmaps(database, stream));
        return index;
    }

Here is the call graph for this function:

Here is the caller graph for this function:

save()

template<uint32_t K, size_t BucketCount, typename Layout >

static Result< void > cuddl::reference_index_file::save ( reference_index< K, BucketCount, Layout > const &  index, reference_database< K, BucketCount, Layout > const &  source, std::filesystem::path const &  path, cuda::stream_ref  stream  )

inlinestatic

Atomically save an already built or loaded index.

Definition at line 78 of file reference_index_file.cuh.

          {
        return [&]() -> Result<void> {
            detail::device_scope const scope{stream};
            CUDDL_TRY(source.view(&index));
            auto database = CUDDL_TRY(snapshot(source, stream));
            auto storage = index.storage();
            auto offsets = CUDDL_TRY(
                download(index.index_offsets_.data(), index.index_offsets_.size(), stream)
            );
            auto posting_count =
                storage == index_storage::dense ? offsets.back() : index.index_posting_capacity_;
            auto postings =
                CUDDL_TRY(download(index.index_postings_.data(), posting_count, stream));
            auto keys =
                CUDDL_TRY(download(index.index_keys_.data(), index.index_keys_.size(), stream));
            auto digest = CUDDL_TRY(detail::reference_database_digest(database));
            detail::database_file_writer writer;
            writer.temporary = path.string() + ".tmp.XXXXXX";
            auto fd = ::mkstemp(writer.temporary.data());
            if (fd < 0) {
                return Err(Error::resource("cannot create index file beside: " + path.string()));
            }
            writer.output = ::fdopen(fd, "wb");
            if (!writer.output) {
                ::close(fd);
                return Err(Error::resource("cannot open temporary index file"));
            }
            CUDDL_TRY(writer.bytes("CUDDLIX\0", 8));
            CUDDL_TRY(writer.value(uint32_t{1}));
            CUDDL_TRY(writer.value(storage == index_storage::dense ? uint32_t{0} : uint32_t{1}));
            CUDDL_TRY(writer.value(digest));
            CUDDL_TRY(writer.value(static_cast<uint64_t>(postings.size())));
            CUDDL_TRY(writer.value(index.pair_work_));
            CUDDL_TRY(writer.words(offsets));
            CUDDL_TRY(writer.words(postings));
            if constexpr (std::endian::native == std::endian::little) {
                CUDDL_TRY(writer.bytes(keys.data(), keys.size() * sizeof(uint16_t)));
            } else {
                for (auto key : keys) {
                    CUDDL_TRY(writer.value(key));
                }
            }
            CUDDL_TRY(writer.value(static_cast<uint32_t>(writer.checksum)));
            if (std::fclose(std::exchange(writer.output, nullptr)) != 0) {
                return Err(Error::resource("cannot close index file"));
            }
            std::filesystem::rename(writer.temporary, path);
            writer.temporary.clear();
            return Ok();
        }();
    } catch (cuda::cuda_error const& error) {
        return Err(Error::cuda(static_cast<cudaError_t>(error.status())));
    } catch (std::system_error const& error) {
        return Err(Error::resource(error.what()));
    } catch (std::bad_alloc const& error) {
        return Err(Error::resource(error.what()));
    }

Here is the call graph for this function:

---

The documentation for this class was generated from the following file:

• reference_index_file.cuh