diff --git a/CHANGELOG.md b/CHANGELOG.md index 219eaad5f6b..7fe7465c6fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ - chore: update benchmark tests to use testing.B.Loop for improved performance ([filecoin-project/lotus#13385](https://github.com/filecoin-project/lotus/pull/13385)) - fix(eth): properly return vm error in all gas estimation methods ([filecoin-project/lotus#13389](https://github.com/filecoin-project/lotus/pull/13389)) - chore: all actor cmd support --actor ([filecoin-project/lotus#13391](https://github.com/filecoin-project/lotus/pull/13391)) +- feat(api): ChainExport API supports exporting a specified Snapshot version ([filecoin-project/lotus#13395](https://github.com/filecoin-project/lotus/pull/13395)) # Node and Miner v1.34.1 / 2025-09-15 @@ -23,7 +24,7 @@ This is a non-critical patch release that fixes an issue with the Lotus `v1.34.0 # Node and Miner v1.34.0 / 2025-09-11 -This is a **MANDATORY Lotus v1.34.0 release**, which will deliver the Filecoin network version 27, codenamed “Golden Week” 🏮. This release candidate sets the upgrade epoch for the Mainnet network to **Epoch 5348280: 2025-09-24T23:00:00Z**. (See the [local time for other timezones](https://www.worldtimebuddy.com/?qm=1&lid=100,5128581,5368361,1816670&h=100&date=2025-9-24&sln=23-24&hf=1&c=1196).) +This is a **MANDATORY Lotus v1.34.0 release**, which will deliver the Filecoin network version 27, codenamed “Golden Week” 🏮. This release candidate sets the upgrade epoch for the Mainnet network to **Epoch 5348280: 2025-09-24T23:00:00Z**. (See the [local time for other timezones](https://www.worldtimebuddy.com/?qm=1&lid=100,5128581,5368361,1816670&h=100&date=2025-9-24&sln=23-24&hf=1&c=1196).) ## ☢️ Upgrade Warnings ☢️ - All Lotus node and Storage Provider (SP) operators must upgrade to v1.34.x before the specified date for the Mainnet network. diff --git a/api/api_full.go b/api/api_full.go index a9dcf7e4f17..f8e9bfb9ce4 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -207,7 +207,7 @@ type FullNode interface { // back to genesis, the entire genesis state, and the most recent 'nroots' // state trees. // If oldmsgskip is set, messages from before the requested roots are also not included. - ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) //perm:read + ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey, version uint64) (<-chan []byte, error) //perm:read // ChainExportRangeInternal triggers the export of a chain // CAR-snapshot directly to disk. It is similar to ChainExport, diff --git a/api/mocks/mock_full.go b/api/mocks/mock_full.go index 86912501d4b..0f19780165b 100644 --- a/api/mocks/mock_full.go +++ b/api/mocks/mock_full.go @@ -142,18 +142,18 @@ func (mr *MockFullNodeMockRecorder) ChainDeleteObj(arg0, arg1 interface{}) *gomo } // ChainExport mocks base method. -func (m *MockFullNode) ChainExport(arg0 context.Context, arg1 abi.ChainEpoch, arg2 bool, arg3 types.TipSetKey) (<-chan []byte, error) { +func (m *MockFullNode) ChainExport(arg0 context.Context, arg1 abi.ChainEpoch, arg2 bool, arg3 types.TipSetKey, arg4 uint64) (<-chan []byte, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ChainExport", arg0, arg1, arg2, arg3) + ret := m.ctrl.Call(m, "ChainExport", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(<-chan []byte) ret1, _ := ret[1].(error) return ret0, ret1 } // ChainExport indicates an expected call of ChainExport. -func (mr *MockFullNodeMockRecorder) ChainExport(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { +func (mr *MockFullNodeMockRecorder) ChainExport(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExport", reflect.TypeOf((*MockFullNode)(nil).ChainExport), arg0, arg1, arg2, arg3) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExport", reflect.TypeOf((*MockFullNode)(nil).ChainExport), arg0, arg1, arg2, arg3, arg4) } // ChainExportRangeInternal mocks base method. diff --git a/api/proxy_gen.go b/api/proxy_gen.go index bfc9fa0d45c..d0b46d2369f 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -120,7 +120,7 @@ type FullNodeMethods struct { ChainDeleteObj func(p0 context.Context, p1 cid.Cid) error `perm:"admin"` - ChainExport func(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey) (<-chan []byte, error) `perm:"read"` + ChainExport func(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey, p4 uint64) (<-chan []byte, error) `perm:"read"` ChainExportRangeInternal func(p0 context.Context, p1 types.TipSetKey, p2 types.TipSetKey, p3 ChainExportConfig) error `perm:"admin"` @@ -1398,14 +1398,14 @@ func (s *FullNodeStub) ChainDeleteObj(p0 context.Context, p1 cid.Cid) error { return ErrNotSupported } -func (s *FullNodeStruct) ChainExport(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey) (<-chan []byte, error) { +func (s *FullNodeStruct) ChainExport(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey, p4 uint64) (<-chan []byte, error) { if s.Internal.ChainExport == nil { return nil, ErrNotSupported } - return s.Internal.ChainExport(p0, p1, p2, p3) + return s.Internal.ChainExport(p0, p1, p2, p3, p4) } -func (s *FullNodeStub) ChainExport(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey) (<-chan []byte, error) { +func (s *FullNodeStub) ChainExport(p0 context.Context, p1 abi.ChainEpoch, p2 bool, p3 types.TipSetKey, p4 uint64) (<-chan []byte, error) { return nil, ErrNotSupported } diff --git a/api/v0api/v1_wrapper.go b/api/v0api/v1_wrapper.go index 97b8ff597d2..dde691bb76b 100644 --- a/api/v0api/v1_wrapper.go +++ b/api/v0api/v1_wrapper.go @@ -213,4 +213,8 @@ func (w *WrapperV1Full) BeaconGetEntry(ctx context.Context, epoch abi.ChainEpoch return w.StateGetBeaconEntry(ctx, epoch) } +func (w *WrapperV1Full) ChainExport(ctx context.Context, nroots abi.ChainEpoch, oldmsgskip bool, tsk types.TipSetKey) (<-chan []byte, error) { + return w.FullNode.ChainExport(ctx, nroots, oldmsgskip, tsk, 2) +} + var _ FullNode = &WrapperV1Full{} diff --git a/chain/lf3/f3.go b/chain/lf3/f3.go index 309c4dd02dd..a7bdaccd0b3 100644 --- a/chain/lf3/f3.go +++ b/chain/lf3/f3.go @@ -21,6 +21,7 @@ import ( "github.com/filecoin-project/go-f3" "github.com/filecoin-project/go-f3/blssig" "github.com/filecoin-project/go-f3/certs" + "github.com/filecoin-project/go-f3/certstore" "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" @@ -41,6 +42,7 @@ type F3Backend interface { Participate(_ context.Context, ticket api.F3ParticipationTicket) (api.F3ParticipationLease, error) ListParticipants() []api.F3Participant GetManifest(ctx context.Context) (*manifest.Manifest, error) + GetCertStore() (*certstore.Store, error) GetCert(ctx context.Context, instance uint64) (*certs.FinalityCertificate, error) GetLatestCert(ctx context.Context) (*certs.FinalityCertificate, error) GetPowerTable(ctx context.Context, tsk types.TipSetKey) (gpbft.PowerEntries, error) @@ -286,6 +288,10 @@ func (fff *F3) Participate(_ context.Context, ticket api.F3ParticipationTicket) return fff.leaser.participate(ticket) } +func (fff *F3) GetCertStore() (*certstore.Store, error) { + return fff.inner.GetCertStore() +} + func (fff *F3) GetCert(ctx context.Context, instance uint64) (*certs.FinalityCertificate, error) { return fff.inner.GetCert(ctx, instance) } diff --git a/cli/chain.go b/cli/chain.go index 7934a794bc7..dfc029ec818 100644 --- a/cli/chain.go +++ b/cli/chain.go @@ -1107,9 +1107,13 @@ var ChainExportCmd = &cli.Command{ &cli.BoolFlag{ Name: "skip-old-msgs", }, + &cli.StringFlag{ + Name: "version", + Value: "2", + }, }, Action: func(cctx *cli.Context) error { - api, closer, err := GetFullNodeAPI(cctx) + api, closer, err := GetFullNodeAPIV1(cctx) if err != nil { return err } @@ -1147,7 +1151,12 @@ var ChainExportCmd = &cli.Command{ return fmt.Errorf("must pass recent stateroots along with skip-old-msgs") } - stream, err := api.ChainExport(ctx, rsrs, skipold, ts.Key()) + version := cctx.Uint64("version") + if version != 1 && version != 2 { + return fmt.Errorf("invalid version %d", version) + } + + stream, err := api.ChainExport(ctx, rsrs, skipold, ts.Key(), version) if err != nil { return err } diff --git a/cli/chain_test.go b/cli/chain_test.go index 76bfcdaefd8..b5f8fcf1358 100644 --- a/cli/chain_test.go +++ b/cli/chain_test.go @@ -495,7 +495,7 @@ func TestChainExport(t *testing.T) { gomock.InOrder( mockApi.EXPECT().ChainHead(ctx).Return(ts, nil), - mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key()).Return(export, nil), + mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key(), uint64(2)).Return(export, nil), ) err := app.Run([]string{"chain", "export", "whatever.car"}) @@ -504,6 +504,35 @@ func TestChainExport(t *testing.T) { assert.Equal(t, expBytes, mockFile.Bytes()) } +func TestChainExportVersionFlag(t *testing.T) { + app, mockApi, _, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainExportCmd)) + defer done() + + mockFile := mockExportFile{new(bytes.Buffer)} + app.Metadata["export-file"] = mockFile + + blk := mock.MkBlock(nil, 0, 0) + ts := mock.TipSet(blk) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + export := make(chan []byte, 2) + export <- []byte("whatever") + export <- []byte{} + close(export) + + gomock.InOrder( + mockApi.EXPECT().ChainHead(ctx).Return(ts, nil), + mockApi.EXPECT().ChainExport(ctx, abi.ChainEpoch(0), false, ts.Key(), uint64(1)).Return(export, nil), + ) + + err := app.Run([]string{"chain", "export", "--version", "1", "whatever.car"}) + assert.NoError(t, err) + + assert.Equal(t, "whatever", mockFile.String()) +} + func TestChainGasPrice(t *testing.T) { app, mockApi, buf, done := NewMockAppWithFullAPI(t, WithCategory("chain", ChainGasPriceCmd)) defer done() diff --git a/cli/mocks_test.go b/cli/mocks_test.go index acb2820630b..69c04a17d88 100644 --- a/cli/mocks_test.go +++ b/cli/mocks_test.go @@ -23,6 +23,7 @@ func NewMockAppWithFullAPI(t *testing.T, cmd *ucli.Command) (*ucli.App, *mocks.M mockFullNode := mocks.NewMockFullNode(ctrl) var fullNode api.FullNode = mockFullNode app.Metadata["test-full-api"] = fullNode + app.Metadata["testnode-full"] = fullNode // this will only work if the implementation uses the app.Writer, // if it uses fmt.*, it has to be refactored diff --git a/documentation/en/api-methods-v1-stable.md b/documentation/en/api-methods-v1-stable.md index 3945e910b0b..34846f8c573 100644 --- a/documentation/en/api-methods-v1-stable.md +++ b/documentation/en/api-methods-v1-stable.md @@ -476,7 +476,8 @@ Inputs: { "/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve" } - ] + ], + 42 ] ``` diff --git a/documentation/en/cli-lotus.md b/documentation/en/cli-lotus.md index 005addc02b9..2338ec88336 100644 --- a/documentation/en/cli-lotus.md +++ b/documentation/en/cli-lotus.md @@ -1970,6 +1970,7 @@ OPTIONS: --tipset value specify tipset to start the export from (default: "@head") --recent-stateroots value specify the number of recent state roots to include in the export (default: 0) --skip-old-msgs (default: false) + --version value (default: "2") --help, -h show help ``` diff --git a/itests/kit/f3.go b/itests/kit/f3.go index 0f58a8cd364..2b5784d5e81 100644 --- a/itests/kit/f3.go +++ b/itests/kit/f3.go @@ -6,6 +6,7 @@ import ( "github.com/filecoin-project/go-f3" "github.com/filecoin-project/go-f3/certs" + "github.com/filecoin-project/go-f3/certstore" "github.com/filecoin-project/go-f3/gpbft" "github.com/filecoin-project/go-f3/manifest" @@ -69,6 +70,10 @@ func (t *MockF3Backend) GetManifest(context.Context) (*manifest.Manifest, error) }, nil } +func (t *MockF3Backend) GetCertStore() (*certstore.Store, error) { + return nil, nil +} + func (t *MockF3Backend) GetCert(_ context.Context, instance uint64) (*certs.FinalityCertificate, error) { if !t.Running { return nil, f3.ErrF3NotRunning diff --git a/node/impl/full/chain.go b/node/impl/full/chain.go index afcc4d58897..82ec8d2caf7 100644 --- a/node/impl/full/chain.go +++ b/node/impl/full/chain.go @@ -31,6 +31,7 @@ import ( "github.com/filecoin-project/go-address" amt4 "github.com/filecoin-project/go-amt-ipld/v4" "github.com/filecoin-project/go-f3/certs" + "github.com/filecoin-project/go-f3/certstore" "github.com/filecoin-project/go-state-types/abi" "github.com/filecoin-project/specs-actors/actors/util/adt" @@ -62,6 +63,7 @@ type ChainModuleAPI interface { ChainGetTipSetAfterHeight(ctx context.Context, h abi.ChainEpoch, tsk types.TipSetKey) (*types.TipSet, error) ChainReadObj(context.Context, cid.Cid) ([]byte, error) ChainGetPath(ctx context.Context, from, to types.TipSetKey) ([]*api.HeadChange, error) + ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipoldmsgs bool, tsk types.TipSetKey, version uint64) (<-chan []byte, error) } var _ ChainModuleAPI = *new(api.FullNode) @@ -703,17 +705,32 @@ func (a ChainAPI) ChainExportRangeInternal(ctx context.Context, head, tail types return nil } -func (a *ChainAPI) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipoldmsgs bool, tsk types.TipSetKey) (<-chan []byte, error) { - ts, err := a.Chain.GetTipSetFromKey(ctx, tsk) +func (m *ChainModule) ChainExport(ctx context.Context, nroots abi.ChainEpoch, skipoldmsgs bool, tsk types.TipSetKey, version uint64) (<-chan []byte, error) { + ts, err := m.Chain.GetTipSetFromKey(ctx, tsk) if err != nil { return nil, xerrors.Errorf("loading tipset %s: %w", tsk, err) } + + var certstore *certstore.Store + if version == 2 { + certstore, err = m.F3.GetCertStore() + if err != nil { + return nil, xerrors.Errorf("getting certstore: %w", err) + } + } + r, w := io.Pipe() out := make(chan []byte) go func() { bw := bufio.NewWriterSize(w, 1<<20) - err = a.Chain.ExportV1(ctx, ts, nroots, skipoldmsgs, bw) + // export v1 snapshot if not certstore or version is SnapshotVersion1 + if certstore == nil || version == 1 { + err = m.Chain.ExportV1(ctx, ts, nroots, skipoldmsgs, bw) + } else { + err = m.Chain.ExportV2(ctx, ts, nroots, skipoldmsgs, certstore, bw) + } + _ = bw.Flush() // it is a write to a pipe _ = w.CloseWithError(err) // it is a pipe }()