diff --git a/README.md b/README.md index fa3b11a6..c69249d8 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,9 @@ ipfs repo version | go-ipfs versions 4 | 0.4.3 - 0.4.5 5 | 0.4.6 - 0.4.10 6 | 0.4.11 - 0.4.15 - 7 | 0.4.16 - current + 7 | 0.4.16 - 0.4.23 + 8 | 0.5.0 - current + 9 | 0.5.0 - current ### How to Run Migrations diff --git a/ipfs-8-to-9/main.go b/ipfs-8-to-9/main.go new file mode 100644 index 00000000..7b407b6c --- /dev/null +++ b/ipfs-8-to-9/main.go @@ -0,0 +1,11 @@ +package main + +import ( + migrate "github.com/ipfs/fs-repo-migrations/go-migrate" + mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration" +) + +func main() { + m := mg8.Migration{} + migrate.Main(&m) +} diff --git a/ipfs-8-to-9/migration/migration.go b/ipfs-8-to-9/migration/migration.go new file mode 100644 index 00000000..56310e39 --- /dev/null +++ b/ipfs-8-to-9/migration/migration.go @@ -0,0 +1,142 @@ +package mg8 + +import ( + base32 "encoding/base32" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + migrate "github.com/ipfs/fs-repo-migrations/go-migrate" + mfsr "github.com/ipfs/fs-repo-migrations/mfsr" + log "github.com/ipfs/fs-repo-migrations/stump" +) + +type Migration struct{} + +func (m Migration) Versions() string { + return "8-to-9" +} + +func (m Migration) Reversible() bool { + return true +} + +const keyFilenamePrefix = "key_" + +const keystoreRoot = "keystore" + +func isEncoded(name string) bool { + _, err := decode(name) + return err == nil +} + +func encode(name string) (string, error) { + if name == "" { + return "", fmt.Errorf("key name must be at least one character") + } + + encoder := base32.StdEncoding.WithPadding(base32.NoPadding) + encodedName := encoder.EncodeToString([]byte(name)) + return keyFilenamePrefix + strings.ToLower(encodedName), nil +} + +func decode(name string) (string, error) { + if !strings.HasPrefix(name, keyFilenamePrefix) { + return "", fmt.Errorf("key's filename has unexpected format") + } + + nameWithoutPrefix := strings.ToUpper(name[len(keyFilenamePrefix):]) + decoder := base32.StdEncoding.WithPadding(base32.NoPadding) + data, err := decoder.DecodeString(nameWithoutPrefix) + + return string(data), err +} + +func (m Migration) Apply(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("applying %s repo migration", m.Versions()) + + err := m.encodeDecode( + opts, + isEncoded, // skip if already encoded + encode, + ) + + if err != nil { + return err + } + + err = mfsr.RepoPath(opts.Path).WriteVersion("9") + if err != nil { + log.Error("failed to update version file to 9") + return err + } + + log.Log("updated version file") + + return nil +} + +func (m Migration) encodeDecode(opts migrate.Options, shouldApplyCodec func(string) bool, codec func(string) (string, error)) error { + keystoreRoot := filepath.Join(opts.Path, keystoreRoot) + fileInfos, err := ioutil.ReadDir(keystoreRoot) + + if err != nil { + return err + } + + for _, info := range fileInfos { + if info.IsDir() { + log.Log("skipping ", info.Name(), " as it is directory!") + continue + } + + if shouldApplyCodec(info.Name()) { + log.Log("skipping ", info.Name(), ". Already in expected format!") + continue + } + + log.VLog("Renaming key's filename: ", info.Name()) + encodedName, err := codec(info.Name()) + if err != nil { + return err + } + + src := filepath.Join(keystoreRoot, info.Name()) + dest := filepath.Join(keystoreRoot, encodedName) + + if err := os.Rename(src, dest); err != nil { + return err + } + } + return nil +} + +func (m Migration) Revert(opts migrate.Options) error { + log.Verbose = opts.Verbose + log.Log("reverting migration") + + err := m.encodeDecode( + opts, + func(name string) bool { + return !isEncoded(name) // skip if not encoded + }, + decode, + ) + + if err != nil { + return err + } + + err = mfsr.RepoPath(opts.Path).WriteVersion("8") + if err != nil { + log.Error("failed to update version file to 8") + return err + } + + log.Log("updated version file") + + return nil +} diff --git a/main.go b/main.go index 2d1b78ad..8a4e22a4 100644 --- a/main.go +++ b/main.go @@ -16,10 +16,12 @@ import ( mg4 "github.com/ipfs/fs-repo-migrations/ipfs-4-to-5/migration" mg5 "github.com/ipfs/fs-repo-migrations/ipfs-5-to-6/migration" mg6 "github.com/ipfs/fs-repo-migrations/ipfs-6-to-7/migration" + mg7 "github.com/ipfs/fs-repo-migrations/ipfs-7-to-8/migration" + mg8 "github.com/ipfs/fs-repo-migrations/ipfs-8-to-9/migration" mfsr "github.com/ipfs/fs-repo-migrations/mfsr" ) -var CurrentVersion = 7 +var CurrentVersion = 9 var migrations = []gomigrate.Migration{ &mg0.Migration{}, @@ -29,6 +31,8 @@ var migrations = []gomigrate.Migration{ &mg4.Migration{}, &mg5.Migration{}, &mg6.Migration{}, + &mg7.Migration{}, + &mg8.Migration{}, } func GetIpfsDir() (string, error) { diff --git a/sharness/Makefile b/sharness/Makefile index 153d65d8..e639829f 100644 --- a/sharness/Makefile +++ b/sharness/Makefile @@ -22,6 +22,7 @@ BINS += bin/ipfs-4-to-5 BINS += bin/ipfs-5-to-6 BINS += bin/ipfs-6-to-7 BINS += bin/ipfs-7-to-8 +BINS += bin/ipfs-8-to-9 BINS += bin/ipfs-update BINS += bin/random-files BINS += bin/go-sleep @@ -38,6 +39,7 @@ IPFS_4_TO_5_SRC = ../ipfs-4-to-5 IPFS_5_TO_6_SRC = ../ipfs-5-to-6 IPFS_6_TO_7_SRC = ../ipfs-6-to-7 IPFS_7_TO_8_SRC = ../ipfs-7-to-8 +IPFS_8_TO_9_SRC = ../ipfs-8-to-9 # User might want to override those on the command line GOFLAGS = @@ -125,6 +127,11 @@ bin/ipfs-7-to-8: $(call find_go_files, $(IPFS_7_TO_8_SRC)) BUILD-OPTIONS @echo "*** installing $@ ***" go build $(GOFLAGS) -o $@ $(IPFS_7_TO_8_SRC) +bin/ipfs-8-to-9: $(call find_go_files, $(IPFS_8_TO_9_SRC)) BUILD-OPTIONS + @echo "*** installing $@ ***" + go build $(GOFLAGS) -o $@ $(IPFS_8_TO_9_SRC) + + BUILD-OPTIONS: FORCE @bin/checkflags '$@' '$(GOFLAGS)' '*** new Go flags ***' diff --git a/sharness/lib/test-lib.sh b/sharness/lib/test-lib.sh index 1bdb5832..94596f6f 100644 --- a/sharness/lib/test-lib.sh +++ b/sharness/lib/test-lib.sh @@ -263,7 +263,7 @@ test_init_ipfs_nd() { test_expect_success "ipfs init succeeds" ' export IPFS_PATH="$(pwd)/.ipfs" && - ipfs init -b=1024 > /dev/null + ipfs init -b=2048 > /dev/null ' test_expect_success "prepare config -- mounting and bootstrap rm" ' diff --git a/sharness/t0030-simple-migration.sh b/sharness/t0030-simple-migration.sh index 6bda8bca..1f48f4d2 100755 --- a/sharness/t0030-simple-migration.sh +++ b/sharness/t0030-simple-migration.sh @@ -13,7 +13,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "7" >expected && + echo "9" >expected && test_cmp expected actual ' @@ -30,7 +30,7 @@ test_expect_success "'fs-repo-migrations -v' works" ' ' test_expect_success "'fs-repo-migrations -v' output looks good" ' - echo "7" >expected && + echo "9" >expected && test_cmp expected actual ' diff --git a/sharness/t0120-migration-7-to-8.sh b/sharness/t0120-migration-7-to-8.sh deleted file mode 100755 index 53fe68bc..00000000 --- a/sharness/t0120-migration-7-to-8.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/sh - -test_description="Test migration 7 to 8" - -. lib/test-lib.sh - -# TODO - -test_done diff --git a/sharness/t0120-migration-7-to-9.sh b/sharness/t0120-migration-7-to-9.sh new file mode 100755 index 00000000..1a7e8517 --- /dev/null +++ b/sharness/t0120-migration-7-to-9.sh @@ -0,0 +1,96 @@ +#!/bin/sh + +test_description="Test migration 7 to 9" + +. lib/test-lib.sh + +# Dist specially built with a v0.5.0-dev-8to9pre1 release +export IPFS_DIST_PATH="/ipfs/QmaaN2kipZfUpRSzwvUeG4Xi3yp1JJB294Vj8pnZ24hesF" +export GOPATH="$(pwd)/gopath" +mkdir -p gopath/bin +export PATH="$(pwd)/../bin:$GOPATH/bin:$PATH" +echo $IPFS_PATH + +test_install_ipfs_nd "v0.4.23" + +test_init_ipfs_nd + +# We need bootstrap addresses to migrate. These are defaults from v0.4.23. +test_expect_success "add bootstrap addresses" ' + test_config_set --json Bootstrap "[ + \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN\", + \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa\", + \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb\", + \"/dnsaddr/bootstrap.libp2p.io/ipfs/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt\", + \"/ip4/104.131.131.82/tcp/4001/ipfs/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ\", + \"/ip4/104.236.179.241/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\", + \"/ip4/128.199.219.111/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\", + \"/ip4/104.236.76.40/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\", + \"/ip4/178.62.158.247/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\", + \"/ip6/2604:a880:1:20::203:d001/tcp/4001/ipfs/QmSoLPppuBtQSGwKDZT2M73ULpjvfd3aZ6ha4oFGL1KrGM\", + \"/ip6/2400:6180:0:d0::151:6001/tcp/4001/ipfs/QmSoLSafTMBsPKadTEgaXctDQVcqN88CNLHXMkTNwMKPnu\", + \"/ip6/2604:a880:800:10::4a:5001/tcp/4001/ipfs/QmSoLV4Bbm51jM9C4gDYZQ9Cy3U6aXMJDAbzgu2fzaDs64\", + \"/ip6/2a03:b0c0:0:1010::23:1001/tcp/4001/ipfs/QmSoLer265NRgSp2LA3dPaeykiS1J6DifTC88f5uVQKNAd\" +]" +' + +test_expect_success "remember old bootstrap" ' + ipfs config Bootstrap | grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_old +' + +# no need to launch daemon as this works offline + +test_expect_success "add some keys to the keystore" ' + ipfs key gen -t rsa thisISKey1 && ipfs key gen -t ed25519 key2 +' + +test_expect_success "ipfs key list" ' + ipfs key list > key_list +' + +test_expect_success "run migration 7 to 8" ' + ipfs-7-to-8 -verbose -path="$IPFS_PATH" > migration_output8 +' + +test_expect_success "run migration 8 to 9" ' + ipfs-8-to-9 -verbose -path="$IPFS_PATH" > migration_output9 +' + +test_expect_success "migration processed keys" ' + grep "thisISKey1" migration_output9 && + grep "key2" migration_output9 +' + +test_expect_success "migrated files exist" ' + [ -f "${IPFS_PATH}/keystore/key_orugs42jknfwk6jr" ] && + [ -f "${IPFS_PATH}/keystore/key_nnsxsmq" ] +' + +test_install_ipfs_nd "v0.5.0-dev-8to9pre2" + +test_expect_success "ipfs key list is the same" ' + ipfs key list > new_key_list + test_cmp key_list new_key_list +' + +test_expect_success "migration revert to 8 succeeds" ' + ipfs-8-to-9 -revert -verbose -path="$IPFS_PATH" > revert_output8 +' + +test_expect_success "migration revert to 7 succeeds" ' + ipfs-7-to-8 -revert -verbose -path="$IPFS_PATH" > revert_output7 +' + +test_install_ipfs_nd "v0.4.23" + +test_expect_success "bootstrap addresses were reverted" ' + ipfs config Bootstrap | grep -E -o "Qm[a-zA-Z0-9]+" | sort > bootstrap_revert + test_cmp bootstrap_old bootstrap_revert +' + +test_expect_success "ipfs key list is the same after revert" ' + ipfs key list > revert_key_list + test_cmp key_list revert_key_list +' + +test_done