6a79e02ebd
Summary: ... in addition to those derived from samples. This could be useful when trade-offs favor an offline trained dictionary that's good for the whole work load, which can involve heavy-weight training, vs. on-the-fly training on samples for each file, which has limitations. This involves some breaking changes to some deeper parts of the new compression API. I'm not concerned about performance because this doesn't touch the per-block parts of the API, just the per-file parts. Bonus: change to CompressionManagerWrapper::FindCompatibleCompressionManager to implement what is likely the preferred behavior. Pull Request resolved: https://github.com/facebook/rocksdb/pull/14253 Test Plan: unit test included Reviewed By: hx235 Differential Revision: D91082208 Pulled By: pdillinger fbshipit-source-id: 1442db65e15c9435437204c19787c96f7a40a207
71 lines
2.7 KiB
C++
71 lines
2.7 KiB
C++
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Creates mixed compressor wrapper which uses multiple compression algorithm
|
|
// within same SST file.
|
|
|
|
#pragma once
|
|
#include <memory>
|
|
#include <vector>
|
|
|
|
#include "rocksdb/advanced_compression.h"
|
|
#include "util/atomic.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class MultiCompressorWrapper : public Compressor {
|
|
public:
|
|
explicit MultiCompressorWrapper(const CompressionOptions& opts);
|
|
|
|
DictConfig GetDictGuidance(CacheEntryRole block_type) const override;
|
|
Slice GetSerializedDict() const override;
|
|
CompressionType GetPreferredCompressionType() const override;
|
|
ManagedWorkingArea ObtainWorkingArea() override;
|
|
std::unique_ptr<Compressor> MaybeCloneSpecialized(
|
|
CacheEntryRole block_type, DictConfigArgs&& dict_config) const override;
|
|
|
|
protected:
|
|
const CompressionOptions opts_;
|
|
std::vector<std::unique_ptr<Compressor>> compressors_;
|
|
};
|
|
|
|
struct RandomMixedCompressor : public MultiCompressorWrapper {
|
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> Clone() const override;
|
|
Status CompressBlock(Slice uncompressed_data, char* compressed_output,
|
|
size_t* compressed_output_size,
|
|
CompressionType* out_compression_type,
|
|
ManagedWorkingArea* wa) override;
|
|
};
|
|
|
|
class RandomMixedCompressionManager : public CompressionManagerWrapper {
|
|
using CompressionManagerWrapper::CompressionManagerWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> GetCompressorForSST(
|
|
const FilterBuildingContext& context, const CompressionOptions& opts,
|
|
CompressionType preferred) override;
|
|
};
|
|
|
|
struct RoundRobinCompressor : public MultiCompressorWrapper {
|
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> Clone() const override;
|
|
Status CompressBlock(Slice uncompressed_data, char* compressed_output,
|
|
size_t* compressed_output_size,
|
|
CompressionType* out_compression_type,
|
|
ManagedWorkingArea* wa) override;
|
|
static RelaxedAtomic<uint64_t> block_counter;
|
|
};
|
|
|
|
class RoundRobinManager : public CompressionManagerWrapper {
|
|
using CompressionManagerWrapper::CompressionManagerWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> GetCompressorForSST(
|
|
const FilterBuildingContext& context, const CompressionOptions& opts,
|
|
CompressionType preferred) override;
|
|
};
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|