Add Peapod to NeoFS Lens (#2514)

CHANGELOG.md

@@ -16,6 +16,7 @@ minor release, the component will be purged, so be prepared (see `Updating` sect
 - New storage component for small objects named Peapod (#2453)
 - New `blobovnicza-to-peapod` tool providing blobovnicza-to-peapod data migration (#2453)
 - SN's version and capacity is announced via the attributes automatically but can be overwritten explicitly (#2455, #602)
+- `peapod` command for `neofs-lens` (#2507)
 
 ### Fixed
 - `neo-go` RPC connection loss handling (#1337)

cmd/neofs-lens/internal/peapod/inspect.go

@@ -0,0 +1,36 @@
+package peapod
+
+import (
+ common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+ blobstorcommon "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+ "github.com/spf13/cobra"
+)
+
+var inspectCMD = &cobra.Command{
+ Use: "inspect",
+ Short: "Object inspection",
+ Long: `Inspect specific object in a Peapod.`,
+ Run: inspectFunc,
+}
+
+func init() {
+ common.AddAddressFlag(inspectCMD, &vAddress)
+ common.AddComponentPathFlag(inspectCMD, &vPath)
+ common.AddOutputFileFlag(inspectCMD, &vOut)
+}
+
+func inspectFunc(cmd *cobra.Command, _ []string) {
+ var getPrm blobstorcommon.GetPrm
+
+ err := getPrm.Address.DecodeString(vAddress)
+ common.ExitOnErr(cmd, common.Errf("failed to decode object address: %w", err))
+
+ ppd := openPeapod(cmd)
+ defer ppd.Close()
+
+ res, err := ppd.Get(getPrm)
+ common.ExitOnErr(cmd, common.Errf("failed to read object from Peapod: %w", err))
+
+ common.PrintObjectHeader(cmd, *res.Object)
+ common.WriteObjectToFile(cmd, vOut, res.RawData)
+}

cmd/neofs-lens/internal/peapod/list.go

@@ -0,0 +1,37 @@
+package peapod
+
+import (
+ "fmt"
+ "io"
+
+ common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+ oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
+ "github.com/spf13/cobra"
+)
+
+var listCMD = &cobra.Command{
+ Use: "list",
+ Short: "Object listing",
+ Long: `List all objects stored in a Peapod.`,
+ Run: listFunc,
+}
+
+func init() {
+ common.AddComponentPathFlag(listCMD, &vPath)
+}
+
+func listFunc(cmd *cobra.Command, _ []string) {
+ // other targets can be supported
+ w := cmd.OutOrStderr()
+
+ wAddr := func(addr oid.Address) error {
+ _, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
+ return err
+ }
+
+ ppd := openPeapod(cmd)
+ defer ppd.Close()
+
+ err := ppd.IterateAddresses(wAddr)
+ common.ExitOnErr(cmd, common.Errf("Peapod iterator failure: %w", err))
+}

cmd/neofs-lens/internal/peapod/root.go

@@ -0,0 +1,41 @@
+package peapod
+
+import (
+ common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
+ "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
+ "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod"
+ "github.com/spf13/cobra"
+)
+
+var (
+ vAddress string
+ vPath string
+ vOut string
+)
+
+// Root defines root command for operations with Peapod.
+var Root = &cobra.Command{
+ Use: "peapod",
+ Short: "Operations with a Peapod",
+}
+
+func init() {
+ Root.AddCommand(listCMD, inspectCMD)
+}
+
+// open and returns read-only peapod.Peapod located in vPath.
+func openPeapod(cmd *cobra.Command) *peapod.Peapod {
+ // interval prm doesn't matter for read-only usage, but must be positive
+ ppd := peapod.New(vPath, 0400, 1)
+ var compressCfg compression.Config
+
+ err := compressCfg.Init()
+ common.ExitOnErr(cmd, common.Errf("failed to init compression config: %w", err))
+
+ ppd.SetCompressor(&compressCfg)
+
+ err = ppd.Open(true)
+ common.ExitOnErr(cmd, common.Errf("failed to open Peapod: %w", err))
+
+ return ppd
+}

cmd/neofs-lens/root.go

@@ -5,6 +5,7 @@ import (
 
  "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/blobovnicza"
  "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/meta"
+ "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/peapod"
  "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal/writecache"
  "github.com/nspcc-dev/neofs-node/misc"
  "github.com/nspcc-dev/neofs-node/pkg/util/gendoc"
@@ -35,6 +36,7 @@ func init() {
  command.SetOut(os.Stdout)
  command.Flags().Bool("version", false, "Application version")
  command.AddCommand(
+ peapod.Root,
  blobovnicza.Root,
  meta.Root,
  writecache.Root,

pkg/local_object_storage/blobstor/peapod/peapod.go

@@ -67,6 +67,13 @@ var errMissingRootBucket = errors.New("missing root bucket")
 // specified permissions.
 //
 // Specified flush interval MUST be positive (see Init).
+//
+// Note that resulting Peapod is NOT ready-to-go:
+// - configure compression first (SetCompressor method)
+// - then open the instance (Open method). Opened Peapod must be finally closed
+// - initialize internal database structure (Init method). May be skipped for read-only usage
+//
+// Any other usage is unsafe and may lead to panic.
 func New(path string, perm fs.FileMode, flushInterval time.Duration) *Peapod {
  if flushInterval <= 0 {
  panic(fmt.Sprintf("non-positive flush interval %v", flushInterval))
@@ -437,6 +444,8 @@ func (x *Peapod) batch(ctx context.Context, fBktRoot func(bktRoot *bbolt.Bucket)
 
 // Iterate iterates over all objects stored in the underlying database and
 // passes them into LazyHandler or Handler. Break on f's false return.
+//
+// Use IterateAddresses to iterate over keys only.
 func (x *Peapod) Iterate(prm common.IteratePrm) (common.IterateRes, error) {
  var addr oid.Address
 
@@ -492,3 +501,31 @@ func (x *Peapod) Iterate(prm common.IteratePrm) (common.IterateRes, error) {
 
  return common.IterateRes{}, nil
 }
+
+// IterateAddresses iterates over all objects stored in the underlying database
+// and passes their addresses into f. If f returns an error, IterateAddresses
+// returns it and breaks.
+func (x *Peapod) IterateAddresses(f func(addr oid.Address) error) error {
+ var addr oid.Address
+
+ err := x.bolt.View(func(tx *bbolt.Tx) error {
+ bktRoot := tx.Bucket(rootBucket)
+ if bktRoot == nil {
+ return errMissingRootBucket
+ }
+
+ return bktRoot.ForEach(func(k, v []byte) error {
+ err := decodeKeyForObject(&addr, k)
+ if err != nil {
+ return fmt.Errorf("decode object address from bucket key: %w", err)
+ }
+
+ return f(addr)
+ })
+ })
+ if err != nil {
+ return fmt.Errorf("exec read-only BoltDB transaction: %w", err)
+ }
+
+ return nil
+}

pkg/local_object_storage/blobstor/peapod/peapod_test.go

@@ -298,3 +298,36 @@ func BenchmarkPeapod_Put(b *testing.B) {
  })
  }
 }
+
+func TestPeapod_IterateAddresses(t *testing.T) {
+ ppd := newTestPeapod(t)
+
+ mSrc := map[oid.Address]struct{}{
+ oidtest.Address(): {},
+ oidtest.Address(): {},
+ oidtest.Address(): {},
+ }
+
+ mDst := make(map[oid.Address]struct{})
+
+ f := func(addr oid.Address) error {
+ mDst[addr] = struct{}{}
+ return nil
+ }
+
+ err := ppd.IterateAddresses(f)
+ require.NoError(t, err)
+ require.Empty(t, mDst)
+
+ for addr := range mSrc {
+ _, err = ppd.Put(common.PutPrm{
+ Address: addr,
+ RawData: nil, // doesn't affect current test
+ })
+ require.NoError(t, err)
+ }
+
+ err = ppd.IterateAddresses(f)
+ require.NoError(t, err)
+ require.Equal(t, mSrc, mDst)
+}