cuddl::reference_database< K, BucketCount, Layout > Class Template Reference

Move-only owner of one immutable contiguous reference database. More...

Public Types
using	layout_type = Layout
using	score_type = uint16_t
using	result_type = reference_search_result
using	batch_result_type = packed_pairwise_counts

Public Member Functions
	reference_database (reference_database const &)=delete
reference_database &	operator= (reference_database const &)=delete
__host__	~reference_database ()
	reference_database (reference_database &&other) noexcept
reference_database &	operator= (reference_database &&other) noexcept
Result< void >	copy_scores_async (device_span< score_type > scores, cuda::stream_ref stream) const
	Writes the reference scores, row-major in reference-ID order, to scores.
std::span< std::string const >	names () const noexcept
	Reference labels in reference-ID order.
reference_database_metadata	metadata () const noexcept
	Compatibility metadata and reference count.
uint32_t	reference_count () const noexcept
	Reference count.
cuda::device_ref	device () const noexcept
	Device holding the rows: the device of the stream that built them.
size_t	persistent_row_bytes () const noexcept
	Bytes this database's own row store needs.
size_t	single_query_workspace_bytes () const noexcept
	Caller-owned bytes for one exhaustive query on this database; always zero.
uint32_t	single_query_result_count () const noexcept
	Results one exhaustive query writes: one per reference.
Result< void >	search_async (device_span< score_type const > query, score_compatibility const &query_compatibility, device_span< uint8_t > workspace, device_span< result_type > results, cuda::stream_ref stream) const
template<typename OnTile >
Result< void >	search_batch_async (device_span< score_type const > queries, score_compatibility const &query_compatibility, uint32_t query_id_offset, device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, cuda::stream_ref stream) const
template<typename OnTile >
Result< void >	search_all_to_all_async (device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, cuda::stream_ref stream, all_to_all_shard shard={}) const
template<typename OnTile >
Result< void >	search_all_to_all_async (device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, cuda::stream_ref stream, all_to_all_tile_queue &tiles) const
	Exhaustive search of the tiles claimed from tiles; see all_to_all_tile_queue.
Result< size_t >	search_workspace_bytes (cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index=nullptr) const
Result< cuddl::batch_search_requirements >	batch_search_requirements (uint32_t query_count, cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index=nullptr) const
Result< cuddl::batch_search_requirements >	all_to_all_search_requirements (reference_index< K, BucketCount, Layout > const *index=nullptr) const
Result< void >	search_async (device_span< score_type const > query, score_compatibility const &query_compatibility, device_span< uint8_t > workspace, device_span< result_type > results, device_span< uint32_t > result_count, search_options options, cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index=nullptr) const
template<typename OnTile >
Result< void >	search_batch_async (device_span< score_type const > queries, score_compatibility const &query_compatibility, uint32_t query_id_offset, device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, search_options options, cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index=nullptr) const
template<typename OnTile >
Result< void >	search_all_to_all_async (device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, search_options options, cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index=nullptr, all_to_all_shard shard={}) const
template<typename OnTile >
Result< void >	search_all_to_all_async (device_span< uint8_t > workspace, device_span< batch_result_type > results, OnTile &&on_tile, device_span< uint32_t > result_match_counts, search_options options, cuda::stream_ref stream, reference_index< K, BucketCount, Layout > const *index, all_to_all_tile_queue &tiles) const
	Indexed or threshold search of the tiles claimed from tiles; see all_to_all_tile_queue.

Static Public Member Functions
static Result< reference_database >	build_async (device_span< score_type const > rows, score_compatibility compatibility, cuda::stream_ref stream)
	Builds a database from flat row-major scores on stream.
static constexpr size_t	persistent_row_bytes (uint32_t reference_count) noexcept
	Bytes one compact row store needs for reference_count references.
static constexpr size_t	single_query_workspace_bytes (uint32_t reference_count) noexcept
static constexpr uint32_t	all_to_all_result_capacity (uint32_t reference_count) noexcept
static constexpr uint32_t	single_query_result_count (uint32_t reference_count) noexcept

Friends
class	reference_index_file
template<uint32_t , size_t , typename >
class	reference_index
class	reference_database_file

Detailed Description

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>
class cuddl::reference_database< K, BucketCount, Layout >

Move-only owner of one immutable contiguous reference database.

Building enqueues row copies on the supplied stream. Inputs and the returned database must remain alive until that stream completes. The allocation stream must outlive the database. Complete work on other streams before destroying or move-assigning the database.

Definition at line 2919 of file reference_database.cuh.

Member Typedef Documentation

batch_result_type

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

using cuddl::reference_database< K, BucketCount, Layout >::batch_result_type = packed_pairwise_counts

Definition at line 2932 of file reference_database.cuh.

layout_type

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

using cuddl::reference_database< K, BucketCount, Layout >::layout_type = Layout

Definition at line 2929 of file reference_database.cuh.

result_type

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

using cuddl::reference_database< K, BucketCount, Layout >::result_type = reference_search_result

Definition at line 2931 of file reference_database.cuh.

score_type

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

using cuddl::reference_database< K, BucketCount, Layout >::score_type = uint16_t

Definition at line 2930 of file reference_database.cuh.

Constructor & Destructor Documentation

reference_database() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

cuddl::reference_database< K, BucketCount, Layout >::reference_database ( reference_database< K, BucketCount, Layout > const &  )

delete

~reference_database()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

__host__ cuddl::reference_database< K, BucketCount, Layout >::~reference_database ( )

inline

Definition at line 2938 of file reference_database.cuh.

{}  // NOLINT(modernize-use-equals-default)

reference_database() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

cuddl::reference_database< K, BucketCount, Layout >::reference_database ( reference_database< K, BucketCount, Layout > &&  other )

inlinenoexcept

Definition at line 2940 of file reference_database.cuh.

        : planes_(std::move(other.planes_)),
          device_(other.device_),
          metadata_(std::exchange(other.metadata_, {})),
          names_(std::move(other.names_)),
          identity_(std::move(other.identity_)),
          deltas_(std::move(other.deltas_)) {}

Member Function Documentation

all_to_all_result_capacity()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static constexpr uint32_t cuddl::reference_database< K, BucketCount, Layout >::all_to_all_result_capacity ( uint32_t  reference_count )

inlinestaticconstexprnoexcept

Definition at line 3065 of file reference_database.cuh.

               {
        return view_type::all_to_all_result_capacity(reference_count);
    }

Here is the call graph for this function:

all_to_all_search_requirements()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< cuddl::batch_search_requirements > cuddl::reference_database< K, BucketCount, Layout >::all_to_all_search_requirements ( reference_index< K, BucketCount, Layout > const *  index = nullptr ) const

inline

Definition at line 3162 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.indexed_all_to_all_search_requirements();
    }

batch_search_requirements()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< cuddl::batch_search_requirements > cuddl::reference_database< K, BucketCount, Layout >::batch_search_requirements ( uint32_t  query_count, cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index = nullptr  ) const

inline

Definition at line 3153 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.indexed_batch_search_requirements(query_count, stream);
    }

build_async()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static Result< reference_database > cuddl::reference_database< K, BucketCount, Layout >::build_async ( device_span< score_type const >  rows, score_compatibility  compatibility, cuda::stream_ref  stream  )

inlinestatic

Builds a database from flat row-major scores on stream.

The database keeps each row only as bit-planes (detail::score_plane_index), the layout every search compares 32 buckets at a time; rows is not retained. Grouping near-duplicate rows (detail::reference_delta_view) waits for stream once.

Definition at line 2965 of file reference_database.cuh.

      {
        detail::device_scope const scope{stream};
        auto const reference_count = CUDDL_TRY(validate_rows(rows, compatibility));
        auto database = CUDDL_CUDA_TRY(reference_database(stream));
        database.metadata_ = {.compatibility = compatibility, .reference_count = reference_count};
        if (!rows.empty()) {
            database.planes_ = CUDDL_CUDA_TRY(
                cuda::make_device_buffer<uint32_t>(
                    stream, stream.device(), rows.size() / 2U, cuda::no_init
                )
            );
            detail::build_score_planes_kernel<BucketCount>
                <<<detail::
                       warp_grid_blocks(static_cast<size_t>(reference_count) * (BucketCount / 32U)),
                   detail::block_size,
                   0,
                   stream.get()>>>(rows.data(), reference_count, database.planes_.data());
            CUDDL_CUDA_TRY(cudaGetLastError());
            // Rows are compared as base references plus changed buckets; see
            // detail::reference_delta_view. Building it waits for the stream.
            if (!detail::ablated("deltas")) {
                database.deltas_ = CUDDL_TRY(
                    detail::build_reference_deltas<BucketCount>(
                        rows.data(), reference_count, stream
                    )
                );
            }
        }
        return Result<reference_database>::ok(std::move(database));
    }

Here is the call graph for this function:

copy_scores_async()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< void > cuddl::reference_database< K, BucketCount, Layout >::copy_scores_async ( device_span< score_type >  scores, cuda::stream_ref  stream  ) const

inline

Writes the reference scores, row-major in reference-ID order, to scores.

Definition at line 3002 of file reference_database.cuh.

                                                                                   {
        detail::device_scope const scope{stream};
        auto const count = static_cast<size_t>(metadata_.reference_count) * BucketCount;
        if (scores.size() < count) {
            return Err(Error::resource("score destination is too small"));
        }
        if (count == 0U) {
            return Ok();
        }
        if (scores.data() == nullptr) {
            return Err(Error::invalid_argument("score destination must be device accessible"));
        }
        detail::decode_score_planes_kernel<BucketCount>
            <<<detail::warp_grid_blocks(
                   static_cast<size_t>(metadata_.reference_count) * (BucketCount / 32U)
               ),
               detail::block_size,
               0,
               stream.get()>>>(planes_.data(), metadata_.reference_count, scores.data());
        return cuda_try(cudaGetLastError());
    }

Here is the call graph for this function:

device()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

cuda::device_ref cuddl::reference_database< K, BucketCount, Layout >::device ( ) const

inlinenoexcept

Device holding the rows: the device of the stream that built them.

Definition at line 3040 of file reference_database.cuh.

                                                         {
        return device_;
    }

Here is the caller graph for this function:

metadata()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

reference_database_metadata cuddl::reference_database< K, BucketCount, Layout >::metadata ( ) const

inlinenoexcept

Compatibility metadata and reference count.

Definition at line 3030 of file reference_database.cuh.

                                                                        {
        return metadata_;
    }

Here is the caller graph for this function:

names()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

std::span< std::string const > cuddl::reference_database< K, BucketCount, Layout >::names ( ) const

inlinenoexcept

Reference labels in reference-ID order.

Definition at line 3025 of file reference_database.cuh.

                                                                  {
        return names_;
    }

operator=() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

reference_database & cuddl::reference_database< K, BucketCount, Layout >::operator= ( reference_database< K, BucketCount, Layout > &&  other )

inlinenoexcept

Definition at line 2948 of file reference_database.cuh.

                                                                       {
        if (this != &other) {
            planes_ = std::move(other.planes_);
            device_ = other.device_;
            metadata_ = std::exchange(other.metadata_, {});
            names_ = std::move(other.names_);
            identity_ = std::move(other.identity_);
            deltas_ = std::move(other.deltas_);
        }
        return *this;
    }

operator=() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

reference_database & cuddl::reference_database< K, BucketCount, Layout >::operator= ( reference_database< K, BucketCount, Layout > const &  )

delete

persistent_row_bytes() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

size_t cuddl::reference_database< K, BucketCount, Layout >::persistent_row_bytes ( ) const

inlinenoexcept

Bytes this database's own row store needs.

Definition at line 3050 of file reference_database.cuh.

                                                               {
        return view().persistent_row_bytes();
    }

Here is the call graph for this function:

persistent_row_bytes() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static constexpr size_t cuddl::reference_database< K, BucketCount, Layout >::persistent_row_bytes ( uint32_t  reference_count )

inlinestaticconstexprnoexcept

Bytes one compact row store needs for reference_count references.

Definition at line 3045 of file reference_database.cuh.

                                                                                                  {
        return view_type::persistent_row_bytes(reference_count);
    }

Here is the call graph for this function:

reference_count()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

uint32_t cuddl::reference_database< K, BucketCount, Layout >::reference_count ( ) const

inlinenoexcept

Reference count.

Definition at line 3035 of file reference_database.cuh.

                                                            {
        return metadata_.reference_count;
    }

Here is the caller graph for this function:

search_all_to_all_async() [1/4]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_all_to_all_async ( device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, cuda::stream_ref  stream, all_to_all_shard  shard = {}  ) const

inline

Definition at line 3116 of file reference_database.cuh.

                                 {}
    ) const {
        return view().search_all_to_all_async(
            workspace, results, std::forward<OnTile>(on_tile), result_match_counts, stream, shard
        );
    }

search_all_to_all_async() [2/4]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_all_to_all_async ( device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, cuda::stream_ref  stream, all_to_all_tile_queue &  tiles  ) const

inline

Exhaustive search of the tiles claimed from tiles; see all_to_all_tile_queue.

Definition at line 3132 of file reference_database.cuh.

            {
        return view().search_all_to_all_async(
            workspace, results, std::forward<OnTile>(on_tile), result_match_counts, stream, tiles
        );
    }

Here is the call graph for this function:

search_all_to_all_async() [3/4]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_all_to_all_async ( device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, search_options  options, cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index, all_to_all_tile_queue &  tiles  ) const

inline

Indexed or threshold search of the tiles claimed from tiles; see all_to_all_tile_queue.

Definition at line 3238 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.search_all_to_all_indexed_async(
            workspace,
            results,
            std::forward<OnTile>(on_tile),
            result_match_counts,
            options,
            stream,
            tiles
        );
    }

search_all_to_all_async() [4/4]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_all_to_all_async ( device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, search_options  options, cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index = nullptr, all_to_all_shard  shard = {}  ) const

inline

Definition at line 3213 of file reference_database.cuh.

                                 {}
    ) const {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.search_all_to_all_indexed_async(
            workspace,
            results,
            std::forward<OnTile>(on_tile),
            result_match_counts,
            options,
            stream,
            shard
        );
    }

search_async() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_async ( device_span< score_type const >  query, score_compatibility const &  query_compatibility, device_span< uint8_t >  workspace, device_span< result_type >  results, cuda::stream_ref  stream  ) const

inline

Definition at line 3082 of file reference_database.cuh.

            {
        return view().search_async(query, query_compatibility, workspace, results, stream);
    }

Here is the call graph for this function:

search_async() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_async ( device_span< score_type const >  query, score_compatibility const &  query_compatibility, device_span< uint8_t >  workspace, device_span< result_type >  results, device_span< uint32_t >  result_count, search_options  options, cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index = nullptr  ) const

inline

Definition at line 3169 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.search_indexed_async(
            query, query_compatibility, workspace, results, result_count, options, stream
        );
    }

search_batch_async() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_batch_async ( device_span< score_type const >  queries, score_compatibility const &  query_compatibility, uint32_t  query_id_offset, device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, cuda::stream_ref  stream  ) const

inline

Definition at line 3093 of file reference_database.cuh.

            {
        return view().search_batch_async(
            queries,
            query_compatibility,
            query_id_offset,
            workspace,
            results,
            std::forward<OnTile>(on_tile),
            result_match_counts,
            stream
        );
    }

Here is the call graph for this function:

search_batch_async() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<typename OnTile >

Result< void > cuddl::reference_database< K, BucketCount, Layout >::search_batch_async ( device_span< score_type const >  queries, score_compatibility const &  query_compatibility, uint32_t  query_id_offset, device_span< uint8_t >  workspace, device_span< batch_result_type >  results, OnTile &&  on_tile, device_span< uint32_t >  result_match_counts, search_options  options, cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index = nullptr  ) const

inline

Definition at line 3186 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.search_batch_indexed_async(
            queries,
            query_compatibility,
            query_id_offset,
            workspace,
            results,
            std::forward<OnTile>(on_tile),
            result_match_counts,
            options,
            stream
        );
    }

search_workspace_bytes()

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

Result< size_t > cuddl::reference_database< K, BucketCount, Layout >::search_workspace_bytes ( cuda::stream_ref  stream, reference_index< K, BucketCount, Layout > const *  index = nullptr  ) const

inline

Definition at line 3145 of file reference_database.cuh.

            {
        auto search_view = CUDDL_TRY(view(index));
        return search_view.indexed_single_query_workspace_bytes(stream);
    }

single_query_result_count() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

uint32_t cuddl::reference_database< K, BucketCount, Layout >::single_query_result_count ( ) const

inlinenoexcept

Results one exhaustive query writes: one per reference.

Definition at line 3078 of file reference_database.cuh.

                                                                      {
        return view().single_query_result_count();
    }

Here is the call graph for this function:

single_query_result_count() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static constexpr uint32_t cuddl::reference_database< K, BucketCount, Layout >::single_query_result_count ( uint32_t  reference_count )

inlinestaticconstexprnoexcept

Definition at line 3071 of file reference_database.cuh.

               {
        return view_type::single_query_result_count(reference_count);
    }

Here is the call graph for this function:

single_query_workspace_bytes() [1/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

size_t cuddl::reference_database< K, BucketCount, Layout >::single_query_workspace_bytes ( ) const

inlinenoexcept

Caller-owned bytes for one exhaustive query on this database; always zero.

Definition at line 3061 of file reference_database.cuh.

                                                                       {
        return view().single_query_workspace_bytes();
    }

Here is the call graph for this function:

single_query_workspace_bytes() [2/2]

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

static constexpr size_t cuddl::reference_database< K, BucketCount, Layout >::single_query_workspace_bytes ( uint32_t  reference_count )

inlinestaticconstexprnoexcept

Definition at line 3054 of file reference_database.cuh.

               {
        return view_type::single_query_workspace_bytes(reference_count);
    }

Here is the call graph for this function:

Friends And Related Symbol Documentation

reference_database_file

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

friend class reference_database_file

friend

Definition at line 2923 of file reference_database.cuh.

reference_index

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

template<uint32_t , size_t , typename >

friend class reference_index

friend

Definition at line 2922 of file reference_database.cuh.

reference_index_file

template<uint32_t K, size_t BucketCount, typename Layout = default_register_layout>

friend class reference_index_file

friend

Definition at line 2920 of file reference_database.cuh.

---

The documentation for this class was generated from the following file:

• reference_database.cuh