Skip to content

Commit 64d9993

Browse files
authored
feat: migration from 15 to 16 (#190)
* chore: init fs-repo-15-to-16 from 14-to-16 mostly copy & paste * feat: append /webrtc-direct if /quic-v1 present context: ipfs/kubo#10463 * chore: bump latest version * test: sharness last this way we see if latest migration e2e passes, before we hit any issues with legacy tooling tests, which is takes second place in priority of things
1 parent 0dbdc6f commit 64d9993

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+1661
-7
lines changed

.github/workflows/test.yml

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
name: Tests
22

33
env:
4-
GO: 1.18
4+
GO: 1.21
55

66
on:
77
push:
@@ -14,12 +14,12 @@ jobs:
1414
name: Tests
1515
runs-on: ubuntu-latest
1616
steps:
17-
- uses: actions/checkout@v3
17+
- uses: actions/checkout@v4
1818
with:
1919
fetch-depth: 2
2020

2121
- name: Set up Go
22-
uses: actions/setup-go@v4
22+
uses: actions/setup-go@v5
2323
with:
2424
go-version: ${{ env.GO }}
2525

Makefile

+5-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ MIG_DIRS = $(shell ls -d fs-repo-*-to-*)
44
IGNORED_DIRS := $(shell cat ignored-migrations)
55
ACTIVE_DIRS := $(filter-out $(IGNORED_DIRS),$(MIG_DIRS))
66

7-
.PHONY: all build clean cmd sharness test test_go test_13_to_14 test_14_to_15
7+
.PHONY: all build clean cmd sharness test test_go test_14_to_15 test_15_to_16
88

99
all: build
1010

@@ -26,7 +26,7 @@ fs-repo-migrations/fs-repo-migrations:
2626
sharness:
2727
make -C sharness
2828

29-
test: test_go sharness test_13_to_14 test_14_to_15
29+
test: test_go test_14_to_15 test_15_to_16 sharness
3030

3131
clean: $(subst fs-repo,clean.fs-repo,$(ACTIVE_DIRS))
3232
@make -C sharness clean
@@ -52,3 +52,6 @@ test_13_to_14:
5252

5353
test_14_to_15:
5454
@cd fs-repo-14-to-15/not-sharness && ./test.sh
55+
56+
test_15_to_16:
57+
@cd fs-repo-15-to-16/test-e2e && ./test.sh

README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,8 @@ Here is the table showing which repo version corresponds to which Kubo version:
5050
| 12 | 0.12.0 - 0.17.0 |
5151
| 13 | 0.18.0 - 0.20.0 |
5252
| 14 | 0.21.0 - 0.22.0 |
53-
| 15 | 0.23.0 - current |
53+
| 15 | 0.23.0 - 0.29.0 |
54+
| 16 | 0.30.0 - current |
5455

5556
### How to Run Migrations
5657

fs-repo-15-to-16/Makefile

+10
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
.PHONY: build clean
2+
3+
build:
4+
go build -mod=vendor
5+
6+
clean:
7+
go clean
8+
9+
test:
10+
@cd test-e2e && ./test.sh
+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
// Package atomicfile provides the ability to write a file with an eventual
2+
// rename on Close (using os.Rename). This allows for a file to always be in a
3+
// consistent state and never represent an in-progress write.
4+
//
5+
// NOTE: `os.Rename` may not be atomic on your operating system.
6+
package atomicfile
7+
8+
import (
9+
"io/ioutil"
10+
"os"
11+
"path/filepath"
12+
)
13+
14+
// File behaves like os.File, but does an atomic rename operation at Close.
15+
type File struct {
16+
*os.File
17+
path string
18+
}
19+
20+
// New creates a new temporary file that will replace the file at the given
21+
// path when Closed.
22+
func New(path string, mode os.FileMode) (*File, error) {
23+
f, err := ioutil.TempFile(filepath.Dir(path), filepath.Base(path))
24+
if err != nil {
25+
return nil, err
26+
}
27+
if err := os.Chmod(f.Name(), mode); err != nil {
28+
f.Close()
29+
os.Remove(f.Name())
30+
return nil, err
31+
}
32+
return &File{File: f, path: path}, nil
33+
}
34+
35+
// Close the file replacing the configured file.
36+
func (f *File) Close() error {
37+
if err := f.File.Close(); err != nil {
38+
os.Remove(f.File.Name())
39+
return err
40+
}
41+
if err := os.Rename(f.Name(), f.path); err != nil {
42+
return err
43+
}
44+
return nil
45+
}
46+
47+
// Abort closes the file and removes it instead of replacing the configured
48+
// file. This is useful if after starting to write to the file you decide you
49+
// don't want it anymore.
50+
func (f *File) Abort() error {
51+
if err := f.File.Close(); err != nil {
52+
os.Remove(f.Name())
53+
return err
54+
}
55+
if err := os.Remove(f.Name()); err != nil {
56+
return err
57+
}
58+
return nil
59+
}

fs-repo-15-to-16/go.mod

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
module github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16
2+
3+
go 1.22
4+
5+
require github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea

fs-repo-15-to-16/go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea h1:lgfk2PMrJI3bh8FflcBTXyNi3rPLqa75J7KcoUfRJmc=
2+
github.com/ipfs/fs-repo-migrations/tools v0.0.0-20211209222258-754a2dcb82ea/go.mod h1:fADeaHKxwS+SKhc52rsL0P1MUcnyK31a9AcaG0KcfY8=

fs-repo-15-to-16/main.go

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package main
2+
3+
import (
4+
mg15 "github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16/migration"
5+
migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
6+
)
7+
8+
func main() {
9+
m := mg15.Migration{}
10+
migrate.Main(m)
11+
}
+231
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,231 @@
1+
// package mg15 contains the code to perform 15-16 repository migration in Kubo.
2+
// This handles the following:
3+
// - Add /webrtc-direct listener if preexisting /udp/ /quic-v1 exists
4+
package mg15
5+
6+
import (
7+
"encoding/json"
8+
"fmt"
9+
"io"
10+
"os"
11+
"path/filepath"
12+
"regexp"
13+
14+
migrate "github.com/ipfs/fs-repo-migrations/tools/go-migrate"
15+
mfsr "github.com/ipfs/fs-repo-migrations/tools/mfsr"
16+
lock "github.com/ipfs/fs-repo-migrations/tools/repolock"
17+
log "github.com/ipfs/fs-repo-migrations/tools/stump"
18+
19+
"github.com/ipfs/fs-repo-migrations/fs-repo-15-to-16/atomicfile"
20+
)
21+
22+
const backupSuffix = ".15-to-16.bak"
23+
24+
// Migration implements the migration described above.
25+
type Migration struct{}
26+
27+
// Versions returns the current version string for this migration.
28+
func (m Migration) Versions() string {
29+
return "15-to-16"
30+
}
31+
32+
// Reversible returns true, as we keep old config around
33+
func (m Migration) Reversible() bool {
34+
return true
35+
}
36+
37+
// Apply update the config.
38+
func (m Migration) Apply(opts migrate.Options) error {
39+
log.Verbose = opts.Verbose
40+
log.Log("applying %s repo migration", m.Versions())
41+
42+
log.VLog("locking repo at %q", opts.Path)
43+
lk, err := lock.Lock2(opts.Path)
44+
if err != nil {
45+
return err
46+
}
47+
defer lk.Close()
48+
49+
repo := mfsr.RepoPath(opts.Path)
50+
51+
log.VLog(" - verifying version is '15'")
52+
if err := repo.CheckVersion("15"); err != nil {
53+
return err
54+
}
55+
56+
log.Log("> Upgrading config to new format")
57+
58+
path := filepath.Join(opts.Path, "config")
59+
in, err := os.Open(path)
60+
if err != nil {
61+
return err
62+
}
63+
64+
// make backup
65+
backup, err := atomicfile.New(path+backupSuffix, 0600)
66+
if err != nil {
67+
return err
68+
}
69+
if _, err := backup.ReadFrom(in); err != nil {
70+
panicOnError(backup.Abort())
71+
return err
72+
}
73+
if _, err := in.Seek(0, io.SeekStart); err != nil {
74+
panicOnError(backup.Abort())
75+
return err
76+
}
77+
78+
// Create a temp file to write the output to on success
79+
out, err := atomicfile.New(path, 0600)
80+
if err != nil {
81+
panicOnError(backup.Abort())
82+
panicOnError(in.Close())
83+
return err
84+
}
85+
86+
if err := convert(in, out); err != nil {
87+
panicOnError(out.Abort())
88+
panicOnError(backup.Abort())
89+
panicOnError(in.Close())
90+
return err
91+
}
92+
93+
if err := in.Close(); err != nil {
94+
panicOnError(out.Abort())
95+
panicOnError(backup.Abort())
96+
}
97+
98+
if err := repo.WriteVersion("16"); err != nil {
99+
log.Error("failed to update version file to 16")
100+
// There was an error so abort writing the output and clean up temp file
101+
panicOnError(out.Abort())
102+
panicOnError(backup.Abort())
103+
return err
104+
} else {
105+
// Write the output and clean up temp file
106+
panicOnError(out.Close())
107+
panicOnError(backup.Close())
108+
}
109+
110+
log.Log("updated version file")
111+
112+
log.Log("Migration 15 to 16 succeeded")
113+
return nil
114+
}
115+
116+
// panicOnError is reserved for checks we can't solve transactionally if an error occurs
117+
func panicOnError(e error) {
118+
if e != nil {
119+
panic(fmt.Errorf("error can't be dealt with transactionally: %w", e))
120+
}
121+
}
122+
123+
func (m Migration) Revert(opts migrate.Options) error {
124+
log.Verbose = opts.Verbose
125+
log.Log("reverting migration")
126+
lk, err := lock.Lock2(opts.Path)
127+
if err != nil {
128+
return err
129+
}
130+
defer lk.Close()
131+
132+
repo := mfsr.RepoPath(opts.Path)
133+
if err := repo.CheckVersion("16"); err != nil {
134+
return err
135+
}
136+
137+
cfg := filepath.Join(opts.Path, "config")
138+
if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
139+
return err
140+
}
141+
142+
if err := repo.WriteVersion("15"); err != nil {
143+
return err
144+
}
145+
if opts.Verbose {
146+
log.Log("lowered version number to 15")
147+
}
148+
149+
return nil
150+
}
151+
152+
var quicRegex = regexp.MustCompilePOSIX("/quic-v1$")
153+
154+
// convert converts the config from one version to another
155+
func convert(in io.Reader, out io.Writer) error {
156+
confMap := make(map[string]any)
157+
if err := json.NewDecoder(in).Decode(&confMap); err != nil {
158+
return err
159+
}
160+
161+
// Append /webrtc-direct listener if /udp/../quic-v1 is present in any of .Addresses fields
162+
if err := func() error {
163+
a, ok := confMap["Addresses"]
164+
if !ok {
165+
return nil
166+
}
167+
addresses, ok := a.(map[string]any)
168+
if !ok {
169+
fmt.Printf("invalid type for .Addresses got %T expected json map; skipping .Addresses\n", a)
170+
return nil
171+
}
172+
173+
for _, addressToRemove := range [...]string{"Swarm", "Announce", "AppendAnnounce", "NoAnnounce"} {
174+
s, ok := addresses[addressToRemove]
175+
if !ok {
176+
continue
177+
}
178+
179+
swarm, ok := s.([]interface{})
180+
if !ok {
181+
fmt.Printf("invalid type for .Addresses.%s got %T expected json array; skipping .Addresses.%s\n", addressToRemove, s, addressToRemove)
182+
continue
183+
}
184+
185+
var newSwarm []interface{}
186+
uniq := map[string]struct{}{}
187+
for _, v := range swarm {
188+
if addr, ok := v.(string); ok {
189+
190+
// if /quic-v1, add /webrtc-direct under the same port
191+
if quicRegex.MatchString(addr) {
192+
newAddr := quicRegex.ReplaceAllString(addr, "/webrtc-direct")
193+
194+
if _, ok := uniq[newAddr]; ok {
195+
continue
196+
}
197+
uniq[newAddr] = struct{}{}
198+
199+
newSwarm = append(newSwarm, newAddr)
200+
}
201+
202+
// keep original addr
203+
if _, ok := uniq[addr]; ok {
204+
continue
205+
}
206+
uniq[addr] = struct{}{}
207+
208+
newSwarm = append(newSwarm, addr)
209+
continue
210+
}
211+
newSwarm = append(newSwarm, v)
212+
}
213+
addresses[addressToRemove] = newSwarm
214+
}
215+
return nil
216+
}(); err != nil {
217+
return err
218+
}
219+
220+
// Save new config
221+
fixed, err := json.MarshalIndent(confMap, "", " ")
222+
if err != nil {
223+
return err
224+
}
225+
226+
if _, err := out.Write(fixed); err != nil {
227+
return err
228+
}
229+
_, err = out.Write([]byte("\n"))
230+
return err
231+
}

fs-repo-15-to-16/test-e2e/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
repotest

0 commit comments

Comments
 (0)