Skip to content

Commit

Permalink
update deal section
Browse files Browse the repository at this point in the history
  • Loading branch information
@LexLuthr
LexLuthr committed Jan 8, 2025
1 parent 3f4d39f commit 05407fc
Show file tree
Hide file tree
Showing 4 changed files with 140 additions and 47 deletions.
102 changes: 78 additions & 24 deletions web/api/webrpc/market.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,7 @@ type StorageDealSummary struct {
Miner string `json:"miner"`
IsLegacy bool `json:"is_legacy"`
Indexed *bool `db:"indexed" json:"indexed"`
IsDDO bool `db:"is_ddo" json:"is_ddo"`
}

func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDealSummary, error) {
Expand All @@ -207,30 +208,83 @@ func (a *WebRPC) StorageDealInfo(ctx context.Context, deal string) (*StorageDeal
if !isLegacy {
var summaries []StorageDealSummary
err = a.deps.DB.Select(ctx, &summaries, `SELECT
md.uuid,
md.sp_id,
md.created_at,
md.signed_proposal_cid,
md.offline,
md.verified,
md.start_epoch,
md.end_epoch,
md.client_peer_id,
md.chain_deal_id,
md.publish_cid,
md.piece_cid,
md.piece_size,
md.fast_retrieval,
md.announce_to_ipni,
md.url,
md.url_headers,
md.error,
mpd.sector_num,
mpm.indexed
FROM market_mk12_deals md
LEFT JOIN market_piece_deal mpd ON mpd.id = md.uuid AND mpd.sp_id = md.sp_id
LEFT JOIN market_piece_metadata mpm ON mpm.piece_cid = md.piece_cid
WHERE md.uuid = $1`, id.String())
deal.uuid,
deal.sp_id,
deal.created_at,
deal.signed_proposal_cid,
deal.offline,
deal.verified,
deal.start_epoch,
deal.end_epoch,
deal.client_peer_id,
deal.chain_deal_id,
deal.publish_cid,
deal.piece_cid,
deal.piece_size,
deal.fast_retrieval,
deal.announce_to_ipni,
deal.url,
deal.url_headers,
deal.error,
mpd.sector_num,
mpm.indexed,
deal.is_ddo -- New column indicating whether the deal is from market_direct_deals
FROM (
-- Query from market_mk12_deals (default, original table)
SELECT
md.uuid,
md.sp_id,
md.created_at,
md.signed_proposal_cid,
md.offline,
md.verified,
md.start_epoch,
md.end_epoch,
md.client_peer_id,
md.chain_deal_id,
md.publish_cid,
md.piece_cid,
md.piece_size,
md.fast_retrieval,
md.announce_to_ipni,
md.url,
md.url_headers,
md.error,
FALSE AS is_ddo -- Not from market_direct_deals
FROM market_mk12_deals md
WHERE md.uuid = $1
UNION ALL
-- Query from market_direct_deals (new table)
SELECT
mdd.uuid,
mdd.sp_id,
mdd.created_at,
'' AS signed_proposal_cid,
mdd.offline,
mdd.verified,
mdd.start_epoch,
mdd.end_epoch,
'' AS client_peer_id,
0 AS chain_deal_id,
'' AS publish_cid,
mdd.piece_cid,
mdd.piece_size,
mdd.fast_retrieval,
mdd.announce_to_ipni,
'' AS url,
'{}' AS url_headers,
'' AS error,
TRUE AS is_ddo -- From market_direct_deals
FROM market_direct_deals mdd
WHERE mdd.uuid = $1
) AS deal
LEFT JOIN market_piece_deal mpd
ON mpd.id = deal.uuid AND mpd.sp_id = deal.sp_id
LEFT JOIN market_piece_metadata mpm
ON mpm.piece_cid = deal.piece_cid;
`, id.String())

if err != nil {
return &StorageDealSummary{}, xerrors.Errorf("select deal summary: %w", err)
Expand Down
82 changes: 59 additions & 23 deletions web/api/webrpc/sector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,13 @@ import (
"time"

"github.com/docker/go-units"
"github.com/filecoin-project/go-state-types/big"
"github.com/samber/lo"
"github.com/snadrus/must"
"golang.org/x/xerrors"

"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/big"

"github.com/filecoin-project/curio/lib/paths"
"github.com/filecoin-project/curio/lib/storiface"
Expand Down Expand Up @@ -115,6 +115,7 @@ type SectorPieceMeta struct {
PieceCid string `db:"piece_cid"`
PieceSize int64 `db:"piece_size"`

DealID string `db:"deal_id"`
DataUrl string `db:"data_url"`
DataRawSize int64 `db:"data_raw_size"`
DeleteOnFinalize bool `db:"data_delete_on_finalize"`
Expand All @@ -138,7 +139,10 @@ type SectorPieceMeta struct {
PieceParkTaskID *int64 `db:"-"`
PieceParkCleanupTaskID *int64 `db:"-"`

IsSnapPiece bool `db:"-"`
IsSnapPiece bool `db:"is_snap"`

MK12Deal bool `db:"boost_deal"`
LegacyDeal bool `db:"legacy_deal"`
}

type FileLocations struct {
Expand Down Expand Up @@ -436,32 +440,64 @@ func (a *WebRPC) SectorInfo(ctx context.Context, sp string, intid int64) (*Secto

var pieces []SectorPieceMeta

// Fetch PoRep pieces
err = a.deps.DB.Select(ctx, &pieces, `SELECT piece_index, piece_cid, piece_size,
data_url, data_raw_size, data_delete_on_finalize,
f05_publish_cid, f05_deal_id, direct_piece_activation_manifest FROM sectors_sdr_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid)
data_url, data_raw_size, data_delete_on_finalize,
f05_publish_cid, f05_deal_id, direct_piece_activation_manifest,
mpd.id AS deal_id, -- Extracted id from market_piece_deal
mpd.boost_deal, -- Retrieved boost_deal from market_piece_deal
mpd.legacy_deal, -- Retrieved legacy_deal from market_piece_deal
is_snap -- New column indicating whether the piece is a snap deal
FROM (
-- Meta table entries (permanent, prioritized)
SELECT meta.piece_num AS piece_index, meta.piece_cid, meta.piece_size,
NULL AS data_url, meta.raw_data_size AS data_raw_size,
NOT meta.requested_keep_data AS data_delete_on_finalize,
meta.f05_publish_cid, meta.f05_deal_id, meta.ddo_pam AS direct_piece_activation_manifest,
meta.sp_id,
NOT sm.is_cc AS is_snap -- is_snap based on is_cc from sectors_meta
FROM sectors_meta_pieces meta
JOIN sectors_meta sm ON meta.sp_id = sm.sp_id AND meta.sector_num = sm.sector_num
WHERE meta.sp_id = $1 AND meta.sector_num = $2
UNION ALL
-- SDR pipeline entries (temporary, non-snap pieces)
SELECT sdr.piece_index, sdr.piece_cid, sdr.piece_size,
sdr.data_url, sdr.data_raw_size, sdr.data_delete_on_finalize,
sdr.f05_publish_cid, sdr.f05_deal_id, sdr.direct_piece_activation_manifest,
sdr.sp_id,
FALSE AS is_snap -- SDR pipeline pieces are never snap deals
FROM sectors_sdr_initial_pieces sdr
WHERE sdr.sp_id = $1 AND sdr.sector_number = $2
AND NOT EXISTS (
SELECT 1
FROM sectors_meta_pieces meta
WHERE meta.sp_id = sdr.sp_id AND meta.piece_cid = sdr.piece_cid
)
UNION ALL
-- Snap pipeline entries (temporary, always snap deals)
SELECT snap.piece_index, snap.piece_cid, snap.piece_size,
snap.data_url, snap.data_raw_size, snap.data_delete_on_finalize,
NULL AS f05_publish_cid, NULL AS f05_deal_id, snap.direct_piece_activation_manifest,
snap.sp_id,
TRUE AS is_snap -- Snap pipeline pieces are always snap deals
FROM sectors_snap_initial_pieces snap
WHERE snap.sp_id = $1 AND snap.sector_number = $2
AND NOT EXISTS (
SELECT 1
FROM sectors_meta_pieces meta
WHERE meta.sp_id = snap.sp_id AND meta.piece_cid = snap.piece_cid
)
) AS combined
LEFT JOIN market_piece_deal mpd
ON combined.sp_id = mpd.sp_id AND combined.piece_cid = mpd.piece_cid;
`, spid, intid)
if err != nil {
return nil, xerrors.Errorf("failed to fetch sector pieces: %w", err)
}

// Fetch SnapDeals pieces
var snapPieces []SectorPieceMeta

err = a.deps.DB.Select(ctx, &snapPieces, `SELECT piece_index, piece_cid, piece_size,
data_url, data_raw_size, data_delete_on_finalize,
NULL as f05_publish_cid, NULL as f05_deal_id, direct_piece_activation_manifest FROM sectors_snap_initial_pieces WHERE sp_id = $1 AND sector_number = $2`, spid, intid)
if err != nil {
return nil, xerrors.Errorf("failed to fetch snap sector pieces: %w", err)
}

// Mark SnapDeals pieces
for i := range snapPieces {
snapPieces[i].IsSnapPiece = true
}

// Combine both slices
pieces = append(pieces, snapPieces...)

for i := range pieces {
pieces[i].StrPieceSize = types.SizeStr(types.NewInt(uint64(pieces[i].PieceSize)))
pieces[i].StrDataRawSize = types.SizeStr(types.NewInt(uint64(pieces[i].DataRawSize)))
Expand Down
1 change: 1 addition & 0 deletions web/static/pages/mk12-deal/deal.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ class DealDetails extends LitElement {
{property: 'Offline', value: entry.offline},
{property: 'Verified', value: entry.verified},
{property: 'Is Legacy', value: entry.is_legacy},
{property: 'Is DDO', value: entry.is_ddo},
{property: 'Start Epoch', value: html`<pretty-epoch .epoch=${entry.start_epoch}></pretty-epoch>`},
{property: 'End Epoch', value: html`<pretty-epoch .epoch=${entry.end_epoch}></pretty-epoch>`},
{property: 'Client Peer ID', value: entry.client_peer_id},
Expand Down
2 changes: 2 additions & 0 deletions web/static/pages/sector/sector-info.mjs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement {
<th>Piece Index</th>
<th>Piece CID</th>
<th>Piece Size</th>
<th>Deal ID</th>
<th>Data URL</th>
<th>Data Raw Size</th>
<th>Delete On Finalize</th>
Expand All @@ -130,6 +131,7 @@ customElements.define('sector-info',class SectorInfo extends LitElement {
<td>${piece.PieceIndex}</td>
<td><a href="/pages/piece/?id=${piece.PieceCid}">${piece.PieceCid}</a></td>
<td>${piece.PieceSize}</td>
<td><a href="/pages/mk12-deal/?id=${piece.DealID}">${piece.DealID}</a></td>
<td>${piece.DataUrl}</td>
<td>${piece.DataRawSize}</td>
<td>${piece.DeleteOnFinalize}</td>
Expand Down

0 comments on commit 05407fc

Please sign in to comment.