fdc2970d37
Summary: Some pieces of follow-up to https://github.com/facebook/rocksdb/issues/13659. _Recommend hiding whitespace for review_ * Add support for instantiating CompressionManagers through CreateFromString/ObjectLibrary. * Pull CompressorCustomAlg and DecompressorCustomAlg out of db_test2, refactor/improvement them a bit, and put them in testutil.h for sharing with db_stress. Switched it from being built on snappy to being built on lz4 so that it can properly test dictionary compression. * Add a custom compression manager for db_stress that uses these, and add to crash test. This depends on the ObjectLibrary stuff because some invocations of db_stress will not be configured with the custom compression manager but will need to access it to read some existing SST files. * Remove some pieces where the concern of setting compression=kZSTD for compatibility purposes had leaked into configuring some tests and compression managers. After https://github.com/facebook/rocksdb/issues/13659 this compatibility concern is contained in the SST building code. * Fix BuiltinDecompressorV2SnappyOnly hiding the (ignored) compression dictionary. SST read logic expects the serialized dictionary to be returned by the decompressor even if it's effectively ignored. Updated DBBlockCacheTest.CacheCompressionDict to cover this case. For follow-up: * Combine custom compression and mixed compression types in a file (not clean/easy without duplicating or majorly refactoring the mixed/random compressor) Pull Request resolved: https://github.com/facebook/rocksdb/pull/13710 Test Plan: unit tests updated Reviewed By: hx235 Differential Revision: D76928974 Pulled By: pdillinger fbshipit-source-id: 772cf9cb048d737699b0e2887c624fb64a68aa8c
67 lines
2.6 KiB
C++
67 lines
2.6 KiB
C++
// Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
// This source code is licensed under both the GPLv2 (found in the
|
|
// COPYING file in the root directory) and Apache 2.0 License
|
|
// (found in the LICENSE.Apache file in the root directory).
|
|
//
|
|
// Creates mixed compressor wrapper which uses multiple compression algorithm
|
|
// within same SST file.
|
|
|
|
#pragma once
|
|
#include <memory>
|
|
#include <vector>
|
|
|
|
#include "compression.h"
|
|
#include "rocksdb/advanced_compression.h"
|
|
|
|
namespace ROCKSDB_NAMESPACE {
|
|
|
|
class MultiCompressorWrapper : public Compressor {
|
|
public:
|
|
explicit MultiCompressorWrapper(const CompressionOptions& opts,
|
|
CompressionDict&& dict = {});
|
|
|
|
size_t GetMaxSampleSizeIfWantDict(CacheEntryRole block_type) const override;
|
|
Slice GetSerializedDict() const override;
|
|
CompressionType GetPreferredCompressionType() const override;
|
|
ManagedWorkingArea ObtainWorkingArea() override;
|
|
std::unique_ptr<Compressor> MaybeCloneSpecialized(
|
|
CacheEntryRole block_type, DictSampleArgs&& dict_samples) override;
|
|
|
|
protected:
|
|
std::vector<std::unique_ptr<Compressor>> compressors_;
|
|
};
|
|
|
|
struct RandomMixedCompressor : public MultiCompressorWrapper {
|
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
const char* Name() const override;
|
|
Status CompressBlock(Slice uncompressed_data, std::string* compressed_output,
|
|
CompressionType* out_compression_type,
|
|
ManagedWorkingArea* wa) override;
|
|
};
|
|
|
|
class RandomMixedCompressionManager : public CompressionManagerWrapper {
|
|
using CompressionManagerWrapper::CompressionManagerWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> GetCompressorForSST(
|
|
const FilterBuildingContext& context, const CompressionOptions& opts,
|
|
CompressionType preferred) override;
|
|
};
|
|
|
|
struct RoundRobinCompressor : public MultiCompressorWrapper {
|
|
using MultiCompressorWrapper::MultiCompressorWrapper;
|
|
const char* Name() const override;
|
|
Status CompressBlock(Slice uncompressed_data, std::string* compressed_output,
|
|
CompressionType* out_compression_type,
|
|
ManagedWorkingArea* wa) override;
|
|
static RelaxedAtomic<uint64_t> block_counter;
|
|
};
|
|
|
|
class RoundRobinManager : public CompressionManagerWrapper {
|
|
using CompressionManagerWrapper::CompressionManagerWrapper;
|
|
const char* Name() const override;
|
|
std::unique_ptr<Compressor> GetCompressorForSST(
|
|
const FilterBuildingContext& context, const CompressionOptions& opts,
|
|
CompressionType preferred) override;
|
|
};
|
|
|
|
} // namespace ROCKSDB_NAMESPACE
|