cuDDL
Loading...
Searching...
No Matches
Classes | Namespaces | Typedefs | Enumerations | Functions | Variables
fastx_sequence_file.hpp File Reference
#include <libdeflate.h>
#include <algorithm>
#include <array>
#include <bit>
#include <condition_variable>
#include <cstdint>
#include <cstdlib>
#include <cstring>
#include <deque>
#include <filesystem>
#include <format>
#include <fstream>
#include <memory>
#include <mutex>
#include <optional>
#include <span>
#include <string>
#include <string_view>
#include <thread>
#include <utility>
#include <vector>
#include <cuddl/detail/ablation.hpp>
#include <cuddl/detail/dna.hpp>
#include <cuddl/error.hpp>
Include dependency graph for fastx_sequence_file.hpp:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Classes

struct  cuddl::detail::fastx_sequence_extent
 A contiguous run of sequence bases (record headers and line breaks excluded). More...
 
struct  cuddl::detail::fastx_packed_record
 One run of bases packed two bits each: its first word and its length in bases. More...
 
class  cuddl::detail::fastx_mapped_file
 Read-only contiguous file payload for in-memory FASTX parsing. More...
 
class  cuddl::detail::fastx_fastq_reader
 FASTQ parser over a contiguous in-memory buffer. More...
 
struct  cuddl::detail::fastx_sequence_file
 
struct  cuddl::detail::bases_packer
 A base packer and the bases it takes per step, a multiple of sixteen up to 64. More...
 
struct  cuddl::detail::decompression_target
 Host-visible buffer a caller offers for decompressed bytes. More...
 
struct  cuddl::detail::decompression_source
 Supplies storage for one file, or a null target when none is available. More...
 
struct  cuddl::detail::shared_gzip_decompressor
 libdeflate handle, one per thread because the object is single-threaded. More...
 
struct  cuddl::detail::compact_impl::compact_shuffle_table
 Shuffle controls per whitespace mask, for the vector paths. More...
 
class  cuddl::detail::fastx_load_pool
 
struct  cuddl::reference_build_statistics
 How a build moved sequence bytes to the device. More...
 

Namespaces

namespace  cuddl
 Batch operations over contiguous device rows.
 
namespace  cuddl::detail
 
namespace  cuddl::detail::compact_impl
 

Typedefs

using cuddl::detail::fastx_line_end_fn = decltype(&fastx_line_end_scalar)
 

Enumerations

enum class  cuddl::detail::fastx_format : uint8_t { cuddl::detail::unknown , cuddl::detail::fasta , cuddl::detail::fastq }
 Detected record format for a FASTX buffer. More...
 

Functions

constexpr bool cuddl::detail::fastx_is_sequence_whitespace (char ch) noexcept
 True for bytes that FASTA sequence consumers conventionally skip between bases.
 
size_t cuddl::detail::fastx_line_end_scalar (std::string_view data, size_t position)
 
fastx_line_end_fn cuddl::detail::resolve_fastx_line_end ()
 Selects the fastest line-end scanner the host supports.
 
size_t cuddl::detail::fastx_line_end (std::string_view data, size_t position)
 First line-end position at or after position using the resolved scanner.
 
uint32_t cuddl::detail::pack_bases16_portable (char const *bases, size_t count, uint32_t &word)
 Packs up to sixteen bases into a two-bit word, base j in bits 2j and 2j+1.
 
uint64_t cuddl::detail::pack_bases_pieces (char const *bases, size_t count, uint32_t *words, size_t width)
 Packs width bases from sixteen-base pieces, for a tail shorter than a full step.
 
uint64_t cuddl::detail::pack_bases_portable (char const *bases, size_t count, uint32_t *words)
 
bases_packer cuddl::detail::resolve_bases_packer ()
 Selects the widest base packer the host supports.
 
void cuddl::detail::pack_fastx_sequence_file (fastx_sequence_file &result, uint32_t k)
 Packs a parsed file's records in place into two-bit words, run by run.
 
libdeflate_result cuddl::detail::decompress_gzip_member (libdeflate_decompressor *decoder, std::string_view input, char *output, size_t capacity, size_t &consumed, size_t &produced)
 Decodes one gzip member, checking structure and ISIZE but not checksums.
 
Result< void > cuddl::detail::gunzip_members_into (std::string_view input, std::string &output, std::string const &path)
 Decompresses gzip members from a mapped buffer into output, appending.
 
Result< size_t > cuddl::detail::gunzip_members_direct (std::string_view input, decompression_target target, bool &insufficient, std::string const &path)
 Decompresses gzip members straight into a caller's buffer.
 
uint64_t cuddl::detail::gzip_decompressed_size (std::string const &path)
 Uncompressed size the gzip trailer advertises, or 0 when it cannot be read.
 
size_t cuddl::detail::gzip_size_hint (std::string_view input)
 
compact_shuffle_table const & cuddl::detail::compact_impl::compact_shuffle () noexcept
 
char * cuddl::detail::compact_sequence_whitespace (char const *first, char const *last, char *out)
 Copies bases from [first, last) to out, dropping sequence whitespace.
 
char const * cuddl::detail::fasta_next_header (char const *first, char const *last)
 First '>' in [first, last) that starts a line, or last.
 
char * cuddl::detail::fasta_sequence_compact (char const *first, char const *last, char *out, char const *&stop)
 Compacts one FASTA record's sequence from first to out, stopping at the next header.
 
std::vector< fastx_sequence_extent > cuddl::detail::fastx_fasta_compact_extents (std::string_view data, char *out)
 Collects FASTA sequence extents with whitespace compacted out, in one pass over data.
 
void cuddl::detail::compact_fastx_sequence_extents (fastx_sequence_file &result)
 Compacts sequence whitespace in place, leaving every extent a run of bases.
 
Result< void > cuddl::detail::parse_fastx_sequence_file (fastx_sequence_file &result, std::string const &path)
 
Result< void > cuddl::detail::pack_fasta_chunks (fastx_sequence_file &result, std::string_view data, char *out, uint32_t k, std::string const &path)
 Compact and pack one cache-sized block at a time, without a genome-sized ASCII write.
 
Result< std::unique_ptr< fastx_sequence_file > > cuddl::detail::finish_fastx_sequence_file (std::unique_ptr< fastx_sequence_file > result, std::string_view data, std::string const &path, bool defer_fasta, uint32_t pack_k)
 Parses, packs or defers the bytes data of a loaded file, which result holds.
 
Result< std::optional< std::string_view > > cuddl::detail::inflate_and_pack (fastx_sequence_file &result, std::string_view input, std::string const &path, decompression_source source, uint32_t k)
 Inflates gzip input in this thread's reusable heap buffer and, for FASTA, packs it there and moves only the packed words into a buffer from source.
 
Result< std::unique_ptr< fastx_sequence_file > > cuddl::detail::decode_fastx_sequence_file (std::unique_ptr< fastx_mapped_file > file, std::string const &path, decompression_source source, bool defer_fasta, uint32_t pack_k=0)
 
Result< std::unique_ptr< fastx_sequence_file > > cuddl::detail::load_fastx_sequence_file (std::string const &path, decompression_source source={}, bool defer_fasta=false, bool read=false, uint32_t pack_k=0)
 

Variables

constexpr size_t cuddl::detail::parallel_read_depth = 64
 Reads a parallel file system needs in flight to give one node its bandwidth: on MOGON's Lustre, 16 threads read 2.9 GB/s of ~1 MB files and 64 threads 6.7 GB/s.