Migration 8-9 - Encoding key filenames with base32

README.md

@@ -44,7 +44,9 @@ ipfs repo version | go-ipfs versions
  4 | 0.4.3 - 0.4.5
  5 | 0.4.6 - 0.4.10
  6 | 0.4.11 - 0.4.15
- 7 | 0.4.16 - current
+ 7 | 0.4.16 - 0.4.23
+ 8 | 0.5.0 - current
+ 9 | 0.5.0 - current
 
 ### How to Run Migrations
 

ipfs-8-to-9/main.go

@@ -0,0 +1,11 @@
+package main
+
+import (
+ migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
+ mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration"
+)
+
+func main() {
+ m := mg8.Migration{}
+ migrate.Main(&m)
+}

ipfs-8-to-9/migration/migration.go

@@ -0,0 +1,142 @@
+package mg8
+
+import (
+ base32 "encoding/base32"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strings"
+
+ migrate "github.com/ipfs/fs-repo-migrations/go-migrate"
+ mfsr "github.com/ipfs/fs-repo-migrations/mfsr"
+ log "github.com/ipfs/fs-repo-migrations/stump"
+)
+
+type Migration struct{}
+
+func (m Migration) Versions() string {
+ return "8-to-9"
+}
+
+func (m Migration) Reversible() bool {
+ return true
+}
+
+const keyFilenamePrefix = "key_"
+
+const keystoreRoot = "keystore"
+
+func isEncoded(name string) bool {
+ _, err := decode(name)
+ return err == nil
+}
+
+func encode(name string) (string, error) {
+ if name == "" {
+ return "", fmt.Errorf("key name must be at least one character")
+ }
+
+ encoder := base32.StdEncoding.WithPadding(base32.NoPadding)
+ encodedName := encoder.EncodeToString([]byte(name))
+ return keyFilenamePrefix + strings.ToLower(encodedName), nil
+}
+
+func decode(name string) (string, error) {
+ if !strings.HasPrefix(name, keyFilenamePrefix) {
+ return "", fmt.Errorf("key's filename has unexpected format")
+ }
+
+ nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):])
+ decoder := base32.StdEncoding.WithPadding(base32.NoPadding)
+ data, err := decoder.DecodeString(nameWithoutPrefix)
+
+ return string(data), err
+}
+
+func (m Migration) Apply(opts migrate.Options) error {
+ log.Verbose = opts.Verbose
+ log.Log("applying %s repo migration", m.Versions())
+
+ err := m.encodeDecode(
+ opts,
+ isEncoded, // skip if already encoded
+ encode,
+ )
+
+ if err != nil {
+ return err
+ }
+
+ err = mfsr.RepoPath(opts.Path).WriteVersion("9")
+ if err != nil {
+ log.Error("failed to update version file to 9")
+ return err
+ }
+
+ log.Log("updated version file")
+
+ return nil
+}
+
+func (m Migration) encodeDecode(opts migrate.Options, shouldApplyCodec func(string) bool, codec func(string) (string, error)) error {
+ keystoreRoot := filepath.Join(opts.Path, keystoreRoot)
+ fileInfos, err := ioutil.ReadDir(keystoreRoot)
+
+ if err != nil {
+ return err
+ }
+
+ for _, info := range fileInfos {
+ if info.IsDir() {
+ log.Log("skipping ", info.Name(), " as it is directory!")
+ continue
+ }
+
+ if shouldApplyCodec(info.Name()) {
+ log.Log("skipping ", info.Name(), ". Already in expected format!")
+ continue
+ }
+
+ log.VLog("Renaming key's filename: ", info.Name())
+ encodedName, err := codec(info.Name())
+ if err != nil {
+ return err
+ }
+
+ src := filepath.Join(keystoreRoot, info.Name())
+ dest := filepath.Join(keystoreRoot, encodedName)
+
+ if err := os.Rename(src, dest); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func (m Migration) Revert(opts migrate.Options) error {
+ log.Verbose = opts.Verbose
+ log.Log("reverting migration")
+
+ err := m.encodeDecode(
+ opts,
+ func(name string) bool {
+ return !isEncoded(name) // skip if not encoded
+ },
+ decode,
+ )
+
+ if err != nil {
+ return err
+ }
+
+ err = mfsr.RepoPath(opts.Path).WriteVersion("8")
+ if err != nil {
+ log.Error("failed to update version file to 8")
+ return err
+ }
+
+ log.Log("updated version file")
+
+ return nil
+}

main.go

@@ -16,10 +16,12 @@ import (
  mg4 "github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/migration"
  mg5 "github.com/ipfs/fs-repo-migrations/ipfs-5-to-6/migration"
  mg6 "github.com/ipfs/fs-repo-migrations/ipfs-6-to-7/migration"
+ mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration"
+ mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration"
  mfsr "github.com/ipfs/fs-repo-migrations/mfsr"
 )
 
-var CurrentVersion = 7
+var CurrentVersion = 9
 
 var migrations = []gomigrate.Migration{
  &mg0.Migration{},
@@ -29,6 +31,8 @@ var migrations = []gomigrate.Migration{
  &mg4.Migration{},
  &mg5.Migration{},
  &mg6.Migration{},
+ &mg7.Migration{},
+ &mg8.Migration{},
 }
 
 func GetIpfsDir() (string, error) {

sharness/Makefile

@@ -22,6 +22,7 @@ BINS += bin/ipfs-4-to-5
 BINS += bin/ipfs-5-to-6
 BINS += bin/ipfs-6-to-7
 BINS += bin/ipfs-7-to-8
+BINS += bin/ipfs-8-to-9
 BINS += bin/ipfs-update
 BINS += bin/random-files
 BINS += bin/go-sleep
@@ -38,6 +39,7 @@ IPFS_4_TO_5_SRC = ../ipfs-4-to-5
 IPFS_5_TO_6_SRC = ../ipfs-5-to-6
 IPFS_6_TO_7_SRC = ../ipfs-6-to-7
 IPFS_7_TO_8_SRC = ../ipfs-7-to-8
+IPFS_8_TO_9_SRC = ../ipfs-8-to-9
 
 # User might want to override those on the command line
 GOFLAGS =
@@ -125,6 +127,11 @@ bin/ipfs-7-to-8: $(call find_go_files, $(IPFS_7_TO_8_SRC)) BUILD-OPTIONS
  @echo "*** installing $@ ***"
  go build $(GOFLAGS) -o $@ $(IPFS_7_TO_8_SRC)
 
+bin/ipfs-8-to-9: $(call find_go_files, $(IPFS_8_TO_9_SRC)) BUILD-OPTIONS
+ @echo "*** installing $@ ***"
+ go build $(GOFLAGS) -o $@ $(IPFS_8_TO_9_SRC)
+
+
 BUILD-OPTIONS: FORCE
  @bin/checkflags '$@' '$(GOFLAGS)' '*** new Go flags ***'
 

sharness/lib/test-lib.sh

@@ -263,7 +263,7 @@ test_init_ipfs_nd() {
 
  test_expect_success "ipfs init succeeds" '
  export IPFS_PATH="$(pwd)/.ipfs" &&
- ipfs init -b=1024 > /dev/null
+ ipfs init -b=2048 > /dev/null
  '
 
  test_expect_success "prepare config -- mounting and bootstrap rm" '

sharness/t0030-simple-migration.sh

@@ -13,7 +13,7 @@ test_expect_success "'fs-repo-migrations -v' works" '
 '
 
 test_expect_success "'fs-repo-migrations -v' output looks good" '
- echo "7" >expected &&
+ echo "9" >expected &&
  test_cmp expected actual
 '
 
@@ -30,7 +30,7 @@ test_expect_success "'fs-repo-migrations -v' works" '
 '
 
 test_expect_success "'fs-repo-migrations -v' output looks good" '
- echo "7" >expected &&
+ echo "9" >expected &&
  test_cmp expected actual
 '
 

sharness/t0120-migration-7-to-9.sh

@@ -0,0 +1,96 @@
+#!/bin/sh
+
+test_description="Test migration 7 to 9"
+
+. lib/test-lib.sh
+
+# Dist specially built with a v0.5.0-dev-8to9pre1 release
+export IPFS_DIST_PATH="/ipfs/QmaaN2kipZfUpRSzwvUeG4Xi3yp1JJB294Vj8pnZ24hesF"
+export GOPATH="$(pwd)/gopath"
+mkdir -p gopath/bin
+export PATH="$(pwd)/../bin:$GOPATH/bin:$PATH"
+echo $IPFS_PATH
+
+test_install_ipfs_nd "v0.4.23"
+
+test_init_ipfs_nd
+
+# We need bootstrap addresses to migrate. These are defaults from v0.4.23.
+test_expect_success "add bootstrap addresses" '
+ test_config_set --json Bootstrap "[
+ \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN\",
+ \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa\",
+ \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb\",
+ \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt\",
+ \"/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ\",
+ \"/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\",
+ \"/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\",
+ \"/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\",
+ \"/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\",
+ \"/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\",
+ \"/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\",
+ \"/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\",
+ \"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\"
+]"
+'
+
+test_expect_success "remember old bootstrap" '
+ ipfs config Bootstrap | grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_old
+'
+
+# no need to launch daemon as this works offline
+
+test_expect_success "add some keys to the keystore" '
+ ipfs key gen -t rsa thisISKey1 && ipfs key gen -t ed25519 key2
+'
+
+test_expect_success "ipfs key list" '
+ ipfs key list > key_list
+'
+
+test_expect_success "run migration 7 to 8" '
+ ipfs-7-to-8 -verbose -path="$IPFS_PATH" > migration_output8
+'
+
+test_expect_success "run migration 8 to 9" '
+ ipfs-8-to-9 -verbose -path="$IPFS_PATH" > migration_output9
+'
+
+test_expect_success "migration processed keys" '
+ grep "thisISKey1" migration_output9 &&
+ grep "key2" migration_output9
+'
+
+test_expect_success "migrated files exist" '
+ [ -f "${IPFS_PATH}/keystore/key_orugs42jknfwk6jr" ] &&
+ [ -f "${IPFS_PATH}/keystore/key_nnsxsmq" ]
+'
+
+test_install_ipfs_nd "v0.5.0-dev-8to9pre2"
+
+test_expect_success "ipfs key list is the same" '
+ ipfs key list > new_key_list
+ test_cmp key_list new_key_list
+'
+
+test_expect_success "migration revert to 8 succeeds" '
+ ipfs-8-to-9 -revert -verbose -path="$IPFS_PATH" > revert_output8
+'
+
+test_expect_success "migration revert to 7 succeeds" '
+ ipfs-7-to-8 -revert -verbose -path="$IPFS_PATH" > revert_output7
+'
+
+test_install_ipfs_nd "v0.4.23"
+
+test_expect_success "bootstrap addresses were reverted" '
+ ipfs config Bootstrap | grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_revert
+ test_cmp bootstrap_old bootstrap_revert
+'
+
+test_expect_success "ipfs key list is the same after revert" '
+ ipfs key list > revert_key_list
+ test_cmp key_list revert_key_list
+'
+
+test_done