commit	b59a8b25398bb18fa22c5873f0ad1072710a80e4	[log] [tgz]
author	Mostyn Bramley-Moore <mostyn@antipode.se>	Tue Oct 19 17:00:19 2021
committer	Mostyn Bramley-Moore <mostyn@antipode.se>	Tue Oct 19 17:00:19 2021
tree	2b1fee1211b80d9ae219717a66ebf2f75dc8e2f7
parent	7c77d5c952c3fb893c5e54bd84a9196da80b2e2a [diff]

tree: 2b1fee1211b80d9ae219717a66ebf2f75dc8e2f7

README.md

zstdpool-syncpool

github.com/klauspost/compress/zstd is a native Go implementation of Zstandard, with an API that leaks memory and goroutines if you don't call Close() on every Encoder and Decoder that you create, and Decoders cannot be reused after they are closed.

zstdpool-syncpool is a Go wrapper for github.com/klauspost/compress/zstd and sync.Pool, which automatically frees resources if you forget to call Close() and/or when items are dropped from the sync.Pool.

Background: https://github.com/klauspost/compress/issues/264

Basic usage

https://pkg.go.dev/github.com/mostynb/zstdpool-syncpool

import (
	"github.com/klauspost/compress/zstd"
	syncpool "github.com/mostynb/zstdpool-syncpool"
)

// Create a sync.Pool which returns wrapped *zstd.Decoder's.
decoderPool := syncpool.NewDecoderPool(zstd.WithDecoderConcurrency(1))

// Get a *DecoderWrapper and use it.
decoder := decoderPool.Get().(*syncpool.DecoderWrapper)
decoder.Reset(compressedDataReader)
_, err = io.Copy(uncompressedDataWriter, decoder)

// Return the decoder to the pool. If we forget this, then the zstd.Decoder
// won't leak resources.
decoderPool.Put(decoder)


// Create a sync.Pool which returns wrapped *zstd.Endoder's.
encoderPool := syncpool.NewEncoderPool(zstd.WithEncoderConcurrency(1))

// Get an *EncoderWrapper and use it.
encoder := encoderPool.Get().(*syncpool.EncoderWrapper)
encoder.Reset(compressedDataWriter)
_, err = io.Copy(encoder, uncompressedDataReader)

// Return the encoder to the pool. If we forget this, then the zstd.Encoder
// won't leak resources.
encoderPool.Put(encoder)

Simplified usage

If you would like to avoid type assertions, you can use NewDecoderPoolWapper and NewEncoderPoolWrapper:

import (
	"github.com/klauspost/compress/zstd"
	syncpool "github.com/mostynb/zstdpool-syncpool"
)

// Create a sync.Pool which returns wrapped *zstd.Decoder's.
decoderPool := syncpool.NewDecoderPoolWrapper(zstd.WithDecoderConcurrency(1))

// Get a *DecoderWrapper and use it.
decoder := decoderPool.Get(compressedDataReader)
_, err = io.Copy(uncompressedDataWriter, decoder)

// Return the decoder to the pool. If we forget this, then the zstd.Decoder
// won't leak resources.
decoderPool.Put(decoder)

// Create a sync.Pool which returns wrapped *zstd.Endoder's.
encoderPool := syncpool.NewEncoderPoolWrapper(zstd.WithEncoderConcurrency(1))

// Get an EncoderWrapper and use it.
encoder := encoderPool.Get(compressedDataWriter)
_, err = io.Copy(encoder, uncompressedDataReader)

// Return the encoder to the pool. If we forget this, then the zstd.Encoder
// won't leak resources.
encoderPool.Put(encoder)

Contributing

Bug reports, feature requests, PRs welcome.

License

Licensed under the Apache License, Version 2.0. See the LICENSE file.