diff --git a/ceres/src/transport/protocol/mod.rs b/ceres/src/transport/protocol/mod.rs index 756bf7606..c4b18a711 100644 --- a/ceres/src/transport/protocol/mod.rs +++ b/ceres/src/transport/protocol/mod.rs @@ -193,9 +193,14 @@ impl SmartSession { )); } let repo = Repo::new(self.repo_path.clone(), false); - storage.save_git_repo(repo.clone().into()).await.map_err(|e| { - ProtocolError::InvalidInput(format!("failed to create import repo: {e}")) - })?; + storage + .save_git_repo(repo.clone().into()) + .await + .map_err(|e| { + ProtocolError::InvalidInput(format!( + "failed to create import repo: {e}" + )) + })?; repo } } diff --git a/common/src/utils.rs b/common/src/utils.rs index 9ab41787e..7d85a5ca3 100644 --- a/common/src/utils.rs +++ b/common/src/utils.rs @@ -183,10 +183,22 @@ mod test { "/third-party/rust/", "/third-party/rust/crates" )); - assert!(!is_strict_path_prefix("/third-party/rust", "/third-party/rust")); - assert!(!is_strict_path_prefix("/third-party/rust", "/third-party/rust_v1")); - assert!(!is_strict_path_prefix("/third-party/rust_v1", "/third-party/rust")); - assert!(!is_strict_path_prefix("/third-party/foo", "/third-party/bar")); + assert!(!is_strict_path_prefix( + "/third-party/rust", + "/third-party/rust" + )); + assert!(!is_strict_path_prefix( + "/third-party/rust", + "/third-party/rust_v1" + )); + assert!(!is_strict_path_prefix( + "/third-party/rust_v1", + "/third-party/rust" + )); + assert!(!is_strict_path_prefix( + "/third-party/foo", + "/third-party/bar" + )); } #[test] @@ -206,18 +218,11 @@ mod test { ), Some("/third-party/rust") ); + assert!(nested_import_repo_conflict("/third-party/foo", existing).is_none()); + assert!(nested_import_repo_conflict("/third-party/rust_v1", existing).is_none()); assert!( - nested_import_repo_conflict("/third-party/foo", existing).is_none() - ); - assert!( - nested_import_repo_conflict("/third-party/rust_v1", existing).is_none() - ); - assert!( - nested_import_repo_conflict( - "/third-party/rust/crates/sw/ay/swayws/1.3.0", - existing - ) - .is_none() + nested_import_repo_conflict("/third-party/rust/crates/sw/ay/swayws/1.3.0", existing) + .is_none() ); } diff --git a/jupiter/src/storage/git_db_storage.rs b/jupiter/src/storage/git_db_storage.rs index c81c817a9..8ae46d61d 100644 --- a/jupiter/src/storage/git_db_storage.rs +++ b/jupiter/src/storage/git_db_storage.rs @@ -7,9 +7,7 @@ use callisto::{ }; use common::{ errors::MegaError, - utils::{ - generate_id, nested_import_repo_conflict_message, - }, + utils::{generate_id, nested_import_repo_conflict_message}, }; use futures::Stream; use sea_orm::{ diff --git a/moon/apps/web/components/AdminGroups/AddMembersDialog.tsx b/moon/apps/web/components/AdminGroups/AddMembersDialog.tsx index 648f32bd2..268b2af3a 100644 --- a/moon/apps/web/components/AdminGroups/AddMembersDialog.tsx +++ b/moon/apps/web/components/AdminGroups/AddMembersDialog.tsx @@ -5,6 +5,7 @@ import { Button, LoadingSpinner } from '@gitmono/ui' import { useAddAdminGroupMembers } from '@/hooks/admin/useAddAdminGroupMembers' import { useAdminGroupMembersList } from '@/hooks/admin/useAdminGroupMembersList' import { useGetSyncMembers } from '@/hooks/useGetSyncMembers' +import { megaUserHandle } from '@/utils/megaUser' interface AddMembersDialogProps { groupId: number | null @@ -35,8 +36,8 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) => const addMembersMutation = useAddAdminGroupMembers() - // Get list of existing member usernames in current group - const existingMemberUsernames = new Set(groupMembersData?.data?.items?.map((member) => member.username) || []) + // Group API stores campsite public ids in the `username` field. + const existingMemberIds = new Set(groupMembersData?.data?.items?.map((member) => member.username) || []) // Fetch members when dialog opens useEffect(() => { @@ -47,8 +48,10 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) => } }, [groupId, refetchMembers]) - const handleMemberToggle = (username: string) => { - setSelectedMembers((prev) => (prev.includes(username) ? prev.filter((u) => u !== username) : [...prev, username])) + const handleMemberToggle = (campsiteUserId: string) => { + setSelectedMembers((prev) => + prev.includes(campsiteUserId) ? prev.filter((u) => u !== campsiteUserId) : [...prev, campsiteUserId] + ) } const handleAddMembersSubmit = async () => { @@ -145,9 +148,7 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) =>
{/* Available members to add */} {(() => { - const availableMembers = members.filter( - (member) => !existingMemberUsernames.has(member.user.username) - ) + const availableMembers = members.filter((member) => !existingMemberIds.has(member.user.id)) return availableMembers.length > 0 ? (
@@ -157,7 +158,8 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) =>
{availableMembers.map((member) => { - const isSelected = selectedMembers.includes(member.user.username) + const isSelected = selectedMembers.includes(member.user.id) + const handle = megaUserHandle(member.user) return (
className={`flex cursor-pointer items-center px-4 py-3 transition-colors hover:bg-gray-100 dark:hover:bg-gray-800 ${ isSelected ? 'border-l-4 border-blue-500 bg-blue-50 dark:bg-blue-900/20' : '' }`} - onClick={() => handleMemberToggle(member.user.username)} + onClick={() => handleMemberToggle(member.user.id)} > { e.stopPropagation() - handleMemberToggle(member.user.username) + handleMemberToggle(member.user.id) }} onClick={(e) => e.stopPropagation()} className='mr-3 h-4 w-4 rounded border-gray-300 text-blue-600 focus:ring-blue-500' @@ -186,7 +188,7 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) =>

{member.user.display_name}

-

@{member.user.username}

+

@{handle}

{/* Already in group members */} {(() => { - const existingMembers = members.filter((member) => existingMemberUsernames.has(member.user.username)) + const existingMembers = members.filter((member) => existingMemberIds.has(member.user.id)) return existingMembers.length > 0 ? (
@@ -223,36 +225,42 @@ export const AddMembersDialog = ({ groupId, onClose }: AddMembersDialogProps) =>
- {existingMembers.map((member) => ( -
-
-
-
- {member.user.display_name} -
-

{member.user.display_name}

-

@{member.user.username}

-
-
- - {member.role} - - - In Group - + {existingMembers.map((member) => { + const handle = megaUserHandle(member.user) + + return ( +
+
+
+
+ {member.user.display_name} +
+

+ {member.user.display_name} +

+

@{handle}

+
+
+ + {member.role} + + + In Group + +
-
- ))} + ) + })}
diff --git a/moon/apps/web/components/AdminGroups/GroupMembersDialog.tsx b/moon/apps/web/components/AdminGroups/GroupMembersDialog.tsx index 656829eaf..58fc28c94 100644 --- a/moon/apps/web/components/AdminGroups/GroupMembersDialog.tsx +++ b/moon/apps/web/components/AdminGroups/GroupMembersDialog.tsx @@ -2,8 +2,10 @@ import React, { useState } from 'react' import { Button, LoadingSpinner, PlusIcon, TrashIcon } from '@gitmono/ui' +import { useMemberMap } from '@/components/Issues/utils/sideEffect' import { useAdminGroupMembersList } from '@/hooks/admin/useAdminGroupMembersList' import { useDeleteAdminGroupMember } from '@/hooks/admin/useDeleteAdminGroupMember' +import { megaUserHandle } from '@/utils/megaUser' import { AddMembersDialog } from './AddMembersDialog' @@ -16,6 +18,7 @@ interface GroupMembersDialogProps { export const GroupMembersDialog = ({ groupId, groupName, onClose }: GroupMembersDialogProps) => { const [deletingUsername, setDeletingUsername] = useState(null) const [showAddMembersDialog, setShowAddMembersDialog] = useState(false) + const memberMap = useMemberMap() // Get current group's member list const { data: groupMembersData, isLoading } = useAdminGroupMembersList(groupId || 0, { @@ -82,35 +85,51 @@ export const GroupMembersDialog = ({ groupId, groupName, onClose }: GroupMembers
) : (
- {members.map((member) => ( -
-
- - {member.username.charAt(0).toUpperCase()} - -
-
-

@{member.username}

-

- Joined: {new Date(member.joined_at * 1000).toLocaleDateString()} -

-
-
- + {members.map((member) => { + const synced = memberMap.get(member.username) + const displayName = megaUserHandle(synced?.user, member.username) + const handle = synced?.user.username || member.username + const avatarUrl = synced?.user.avatar_urls?.sm || synced?.user.avatar_urls?.base + const initial = (displayName || handle).charAt(0).toUpperCase() + + return ( +
+ {avatarUrl ? ( + {displayName} + ) : ( +
+ {initial} +
+ )} +
+

{displayName}

+

+ @{handle} + {' · '} + Joined: {new Date(member.joined_at * 1000).toLocaleDateString()} +

+
+
+ +
-
- ))} + ) + })}
)}
diff --git a/moon/apps/web/components/ClView/index.tsx b/moon/apps/web/components/ClView/index.tsx index 0bdb3d50b..0b08b718d 100644 --- a/moon/apps/web/components/ClView/index.tsx +++ b/moon/apps/web/components/ClView/index.tsx @@ -230,7 +230,14 @@ export default function CLView() { {' '} by{' '} - + + m.user.id === item.author || m.user.username === item.author || m.user.github_login === item.author + )?.user.username || item.author + } + > {authorHandle(item.author)} {item.author_is_bot ? ( @@ -251,7 +258,14 @@ export default function CLView() { {' '} by{' '} - + + m.user.id === item.author || m.user.username === item.author || m.user.github_login === item.author + )?.user.username || item.author + } + > {authorHandle(item.author)} {item.author_is_bot ? ( @@ -267,7 +281,16 @@ export default function CLView() { return ( <> by{' '} - + + m.user.id === item.author || + m.user.username === item.author || + m.user.github_login === item.author + )?.user.username || item.author + } + > {authorHandle(item.author)} {item.author_is_bot ? ( @@ -291,7 +314,14 @@ export default function CLView() { return ( <> by{' '} - + + m.user.id === item.author || m.user.username === item.author || m.user.github_login === item.author + )?.user.username || item.author + } + > {authorHandle(item.author)} {item.author_is_bot ? ( diff --git a/moon/apps/web/components/CodeView/BlobView/CodeContent.tsx b/moon/apps/web/components/CodeView/BlobView/CodeContent.tsx index f1ed282a3..9c931367a 100644 --- a/moon/apps/web/components/CodeView/BlobView/CodeContent.tsx +++ b/moon/apps/web/components/CodeView/BlobView/CodeContent.tsx @@ -12,7 +12,8 @@ import { UsersIcon } from '@gitmono/ui' import ThemedMarkdown from '@/components/Theme/ThemedMarkdown/index' import { useGetBlame } from '@/hooks/useGetBlame' -import { useGetOrganizationMember } from '@/hooks/useGetOrganizationMember' +import { useMemberByActor } from '@/hooks/useMemberByActor' +import { megaUserHandle } from '@/utils/megaUser' import { getLanguageForFile } from '@/utils/shikiLanguageFallback' import BlobEditor from './BlobEditor' @@ -21,13 +22,15 @@ import styles from './CodeContent.module.css' type ViewMode = 'code' | 'blame' | 'preview' const UserAvatar = React.memo(({ username, zIndex }: { username?: string; zIndex?: number }) => { - const { data: memberData } = useGetOrganizationMember({ username }) + const { data: member } = useMemberByActor(username) + const displayName = megaUserHandle(member?.user, username || '') + const avatarUrl = member?.user?.avatar_urls?.sm || member?.user?.avatar_urls?.base || '' return ( diff --git a/moon/apps/web/components/CodeView/CommitHistory.tsx b/moon/apps/web/components/CodeView/CommitHistory.tsx index 431a5e3f8..b6eb95edc 100644 --- a/moon/apps/web/components/CodeView/CommitHistory.tsx +++ b/moon/apps/web/components/CodeView/CommitHistory.tsx @@ -7,7 +7,8 @@ import { Avatar, Button, ClockIcon, EyeIcon } from '@gitmono/ui' import { MemberHovercard } from '@/components/InlinePost/MemberHovercard' import { useGetLatestCommit } from '@/hooks/useGetLatestCommit' -import { useGetOrganizationMember } from '@/hooks/useGetOrganizationMember' +import { useMemberByActor } from '@/hooks/useMemberByActor' +import { megaUserHandle } from '@/utils/megaUser' const CommitHyStyle = { width: '100%', @@ -23,7 +24,9 @@ interface CommitHistoryProps { export default function CommitHistory({ flag, path, refs }: CommitHistoryProps) { const [Expand, setExpand] = useState(false) const { data: commitData } = useGetLatestCommit(path, refs) - const { data: memberData } = useGetOrganizationMember({ username: commitData?.author, enabled: !!commitData?.author }) + const { data: member } = useMemberByActor(commitData?.author) + const displayName = megaUserHandle(member?.user, commitData?.author || '') + const hoverUsername = member?.user.username || commitData?.author || '' const ExpandDetails = () => { setExpand(!Expand) @@ -43,10 +46,10 @@ export default function CommitHistory({ flag, path, refs }: CommitHistoryProps) <>
- + - - {commit.author} + + {displayName || commit.author} {commit.short_message} @@ -90,8 +93,8 @@ export default function CommitHistory({ flag, path, refs }: CommitHistoryProps) {Expand && commitData && (

- Signed-off-by: {commitData.author} {'<'} - {memberData?.user?.email} + Signed-off-by: {displayName || commitData.author} {'<'} + {member?.user?.email} {'>'}

)} diff --git a/moon/apps/web/components/CodeView/CommitsView/detail/index.tsx b/moon/apps/web/components/CodeView/CommitsView/detail/index.tsx index 5686a2c49..1786949f4 100644 --- a/moon/apps/web/components/CodeView/CommitsView/detail/index.tsx +++ b/moon/apps/web/components/CodeView/CommitsView/detail/index.tsx @@ -9,7 +9,7 @@ import toast from 'react-hot-toast' import { CommitSummary, CommonPageDiffItemSchema, CommonResultVecMuiTreeNode, DiffItemSchema } from '@gitmono/types' import { LoadingSpinner } from '@gitmono/ui' -import { formatAssignees } from '@/components/CodeView/CommitsView' +import { FormatAssignees } from '@/components/CodeView/CommitsView' import { commitPath } from '@/components/CodeView/CommitsView/items' import FileDiff from '@/components/DiffView/FileDiff' import { MemberHoverAvatarList } from '@/components/Issues/MemberHoverAvatarList' @@ -134,7 +134,7 @@ export const CommitsDetailView: React.FC = () => {
- {formatAssignees([commitsDetail.commit.author])} + authored {commitDate && (
- {formatAssignees([item.committer])} + authored{' '} {formatDistance(fromUnixTime(parseInt(item.date, 10)), new Date(), { diff --git a/moon/apps/web/components/DiffView/comment/CommentThread.tsx b/moon/apps/web/components/DiffView/comment/CommentThread.tsx index 7edf0e066..a11108b03 100644 --- a/moon/apps/web/components/DiffView/comment/CommentThread.tsx +++ b/moon/apps/web/components/DiffView/comment/CommentThread.tsx @@ -5,7 +5,8 @@ import type { CommentReviewResponse, ThreadReviewResponse } from '@gitmono/types import { Avatar, Button } from '@gitmono/ui' import { useGetCurrentUser } from '@/hooks/useGetCurrentUser' -import { useGetOrganizationMember } from '@/hooks/useGetOrganizationMember' +import { useMemberByActor } from '@/hooks/useMemberByActor' +import { megaUserHandle } from '@/utils/megaUser' import { useDeleteComment } from '../hooks/useDeleteComment' import { useDeleteThread } from '../hooks/useDeleteThread' @@ -15,10 +16,11 @@ import { useResolveThread } from '../hooks/useResolveThread' import { useUpdateComment } from '../hooks/useUpdateComment' function UserAvatar({ username, size = 'sm' }: { username: string; size?: 'xs' | 'sm' }) { - const { data: member } = useGetOrganizationMember({ username }) - const avatarUrl = member?.user?.avatar_url + const { data: member } = useMemberByActor(username) + const displayName = megaUserHandle(member?.user, username) + const avatarUrl = member?.user?.avatar_urls?.sm || member?.user?.avatar_urls?.base - return + return } interface CommentItemProps { @@ -304,6 +306,8 @@ function CommentItem({ isUpdating }: CommentItemProps) { const [showMenu, setShowMenu] = useState(false) + const { data: member } = useMemberByActor(comment.user_name) + const displayName = megaUserHandle(member?.user, comment.user_name) return (
@@ -311,7 +315,7 @@ function CommentItem({
- {comment.user_name} + {displayName} {comment.created_at}
{(onEdit || onDelete) && ( diff --git a/moon/apps/web/components/Issues/IssuesContent.tsx b/moon/apps/web/components/Issues/IssuesContent.tsx index 191aaed7d..15104c34c 100644 --- a/moon/apps/web/components/Issues/IssuesContent.tsx +++ b/moon/apps/web/components/Issues/IssuesContent.tsx @@ -36,6 +36,7 @@ import { useGetLabelList } from '@/hooks/useGetLabelList' import { useSyncedMembers } from '@/hooks/useSyncedMembers' import { apiErrorToast } from '@/utils/apiErrorToast' import { atomWithWebStorage } from '@/utils/atomWithWebStorage' +import { megaUserHandle } from '@/utils/megaUser' import { Pagination } from './Pagenation' @@ -63,6 +64,17 @@ export function IssuesContent({ setFilterQuery, shouldClearFilters, setShouldCle const { mutate: issueLists } = useGetIssueLists() const { members } = useSyncedMembers() + const authorHandle = useCallback( + (author: string) => { + const member = members.find( + (m) => m.user.id === author || m.user.username === author || m.user.github_login === author + ) + + return megaUserHandle(member?.user, author) + }, + [members] + ) + const filterState = useFilterState({ scope: scope as string, type: 'issue' }) const filterStateRef = useRef(filterState) @@ -245,10 +257,15 @@ export function IssuesContent({ setFilterQuery, shouldClearFilters, setShouldCle const getIssueDescription = (item: ItemsType[number]) => { const normalizedStatus = item.status.toLowerCase() + const displayAuthor = authorHandle(item.author) + const hoverUsername = + members.find( + (m) => m.user.id === item.author || m.user.username === item.author || m.user.github_login === item.author + )?.user.username || item.author const authorNode = ( <> - - {item.author} + + {displayAuthor} {item.author_is_bot ? ( <> diff --git a/moon/apps/web/components/Issues/MemberHoverAvatarList.tsx b/moon/apps/web/components/Issues/MemberHoverAvatarList.tsx index 0fe370088..7d9f2eb6f 100644 --- a/moon/apps/web/components/Issues/MemberHoverAvatarList.tsx +++ b/moon/apps/web/components/Issues/MemberHoverAvatarList.tsx @@ -1,83 +1,37 @@ import React from 'react' import { Avatar, AvatarStack } from '@primer/react' -import { useQueries } from '@tanstack/react-query' -import { OrganizationMember } from '@gitmono/types/generated' +import { SyncOrganizationMember } from '@gitmono/types/generated' -import { useScope } from '@/contexts/scope' -import { apiClient } from '@/utils/queryClient' - -import { MemberHovercard } from './MemberHoverCardNE' +import { MemberHovercard } from '@/components/InlinePost/MemberHovercard' +import { useMemberMap } from '@/components/Issues/utils/sideEffect' interface MemberHoverAvatarListProps { isLeft?: boolean + /** Actor keys: campsite_user_id, username, or github_login */ authors: string[] } -export const MemberHoverAvatarList = ({ authors, isLeft }: MemberHoverAvatarListProps) => { - const shouldFetch = authors.length > 0 - const query = apiClient.organizations.getMembersByUsername() - const { scope } = useScope() - - const queries = useQueries({ - queries: authors.map((u) => ({ - queryKey: query.requestKey(`${scope}`, `${u}`), - queryFn: () => query.request(`${scope}`, `${u}`), - enabled: shouldFetch - })), - combine: (res) => { - return { - data: res.map((r) => r.data), - pending: res.some((r) => r.isPending) - } - } - }) +export const MemberHoverAvatarList = ({ authors, isLeft }: MemberHoverAvatarListProps) => { + const memberMap = useMemberMap() - return ( - <> - - {queries.pending - ? Array.from({ length: authors.length }).map((_, i) => ( - // eslint-disable-next-line react/no-array-index-key -
- )) - : queries.data.map( - (q) => - q && ( - - ) - )} - - - ) -} + const members = authors + .map((actor) => memberMap.get(actor) as SyncOrganizationMember | undefined) + .filter((m): m is SyncOrganizationMember => !!m) -interface HoverProps { - username: string - userData: OrganizationMember -} -const AvatarwithHover = ({ - src, - hoverProps, - className, - style -}: { - src: string - hoverProps: HoverProps - className?: string - style?: React.CSSProperties -}) => { return ( - <> - -
- -
- - + + {members.map((member) => { + const src = member.user.avatar_urls?.sm || member.user.avatar_urls?.base || '' + + return ( + +
+ +
+ + ) + })} + ) } diff --git a/scripts/crates-sync/Dockerfile b/scripts/crates-sync/Dockerfile index e1a79f377..ea3eff5ba 100644 --- a/scripts/crates-sync/Dockerfile +++ b/scripts/crates-sync/Dockerfile @@ -18,7 +18,10 @@ RUN sed -i 's|http://deb.debian.org|https://deb.debian.org|g' /etc/apt/sources.l WORKDIR /app +# So kubectl logs shows print() immediately (stdout is a pipe, not a TTY). +ENV PYTHONUNBUFFERED=1 + # Build context must be the scripts/crates-sync directory (this folder). COPY . scripts/crates-sync/ -ENTRYPOINT ["python3", "scripts/crates-sync/run_job.py"] +ENTRYPOINT ["python3", "-u", "scripts/crates-sync/run_job.py"] diff --git a/scripts/crates-sync/README.md b/scripts/crates-sync/README.md index 19a58fe9f..049879753 100644 --- a/scripts/crates-sync/README.md +++ b/scripts/crates-sync/README.md @@ -107,6 +107,8 @@ For each `crate@version`, the manifest tracks: On startup: - If a crate version already has `status=ok` in the manifest, it is **skipped by default**. +- If Mega already has history on that path (prior Job), the importer **skips** download/push (`ls-remote`), counts it as skip, and writes `status=ok` so the next run is a cheap manifest skip — unless you pass `--force` / `--force-with-lease` with `--reimport-ok` to overwrite. +- A non-fast-forward push is also treated as already-present (ok), not fail. - `--force` / `--force-with-lease` only affect `git push`; they **do not** disable manifest skipping (so you can use them for non-fast-forward mirrors without re-importing every crate). - To intentionally re-import versions that are already `ok`, use `--reimport-ok`. @@ -208,7 +210,11 @@ Notes: CI builds `mega/crates-sync` from `scripts/crates-sync/Dockerfile` (workflow: `.github/workflows/crates-sync-deploy.yml`). -The Job entrypoint is `run_job.py`: wait for mono → `bootstrap-init` bot token → optional `git pull` on index → `crates-sync.py` with `--keep-crate-cache` and `--max-versions-per-crate 0` (all versions). +The Job entrypoint is `run_job.py`: wait for mono → `bootstrap-init` bot token → +`crates-sync.py` with `--keep-crate-cache`, `--readonly-crate-cache`, and +`--max-versions-per-crate 0` (all versions). Index `git pull` is **off by default** +(`--pull-index` to opt in). Freighter owns refreshing `crates.io-index` and `.crate` files; +the Job only reads them and writes under `mega-crates-work/`. On **mega-rust**, data is expected on the node hostPath: diff --git a/scripts/crates-sync/crates-sync.py b/scripts/crates-sync/crates-sync.py index 7b9fe09fd..cba13792f 100644 --- a/scripts/crates-sync/crates-sync.py +++ b/scripts/crates-sync/crates-sync.py @@ -45,7 +45,7 @@ _waiting_push: set[str] = set() STATUS_HEARTBEAT = False -STATUS_HEARTBEAT_INTERVAL_S = 15.0 +STATUS_HEARTBEAT_INTERVAL_S = 2.0 STATUS_STICKY = False _push_ok_lock = threading.Lock() @@ -116,6 +116,10 @@ def _pushes_per_min_since_start() -> float: ok_total, fail_total = _push_totals() return (ok_total + fail_total) / max(1e-6, mins) +def _pushes_per_sec_last_60s() -> float: + """Successful push rate over the trailing 60s window (ok_60s / 60).""" + return _push_ok_last_60s() / 60.0 + def _progress_reset() -> None: global _progress_index_crates, _progress_versions_queued, _progress_versions_done global _progress_ok, _progress_skip, _progress_fail @@ -211,11 +215,12 @@ def _format_progress_line() -> str: bar = _format_progress_bar(done, denom) scan_tag = "scan=done" if scan_done else "scan=running" count_s = f"{done}/{denom}" if denom else f"done={done}" + pps = _pushes_per_sec_last_60s() return ( f"progress: {bar} {count_s} " f"jobs={jobs} ok={ok_n} skip={skip_n} fail={fail_n} " f"queue={qdepth} crates={crates} {scan_tag} " - f"{_format_eta(done, denom)}" + f"push/s={pps:.2f} {_format_eta(done, denom)}" ) def _clear_status_block_locked() -> None: @@ -260,6 +265,7 @@ def _format_status_block() -> list[str]: fail60 = _push_fail_last_60s() ok_total, fail_total = _push_totals() ppm = _pushes_per_min_since_start() + pps = _pushes_per_sec_last_60s() with _progress_lock: done = _progress_versions_done queued = _progress_versions_queued @@ -279,9 +285,15 @@ def _format_status_block() -> list[str]: except NotImplementedError: qdepth = max(0, queued - done) if denom: - head = f"progress: {_format_progress_bar(done, denom)} {done}/{denom} {_format_eta(done, denom)}" + head = ( + f"progress: {_format_progress_bar(done, denom)} {done}/{denom} " + f"push/s={pps:.2f} {_format_eta(done, denom)}" + ) else: - head = f"progress: {_format_progress_bar(0, None)} scanning index..." + head = ( + f"progress: {_format_progress_bar(0, None)} scanning index... " + f"push/s={pps:.2f}" + ) scan_line = ( f"scan: crates={crates} versions_found={queued} " f"status={'done' if scan_done else 'running (denom grows until full index walk)'}" @@ -295,7 +307,7 @@ def _format_status_block() -> list[str]: ( f"push: ok_60s={ok60} fail_60s={fail60} " f"ok_total={ok_total} fail_total={fail_total} " - f"per_min={ppm:.2f}" + f"per_s={pps:.2f} per_min={ppm:.2f}" ), ] @@ -341,6 +353,8 @@ def _heartbeat_thread(stop_evt: threading.Event) -> None: def _log(level: str, msg: str) -> None: # Standardized, low-noise logging. Use --verbose for command outputs. # Always write to stderr so sticky status (also on stderr) and logs share one stream. + # Progress counters refresh on the heartbeat (~2s). Per-crate OK lines are verbose-only, + # so sticky re-paint here is only for occasional INFO/WARN (and verbose OK). if level == "INFO": c = BLUE elif level == "WARN": @@ -356,7 +370,6 @@ def _log(level: str, msg: str) -> None: _clear_status_block_locked() print(line, file=sys.stderr, flush=True) if STATUS_STICKY: - # Re-paint with fresh progress so OK/INFO lines don't leave a stale block. _render_status_block_locked(_format_status_block()) def info(msg: str) -> None: @@ -521,14 +534,33 @@ def _download_with_progress(url: str, dest_path: str, *, label: str) -> bool: with _download_state_lock: _active_downloads.discard(label) -def check_and_download_crate(crates_dir, crate_name, crate_version, dl_base_url) -> str | None: +def check_and_download_crate( + crates_dir, + crate_name, + crate_version, + dl_base_url, + *, + readonly_cache: bool = False, +) -> str | None: # Construct the filename and path for the crate crate_filename = f"{crate_name}-{crate_version}.crate" crate_path = os.path.join(crates_dir, crate_name, crate_filename) + label = _fmt_repo(crate_name, crate_version) + + # Freighter / shared cache: never create dirs, never download, never delete. + # Missing or invalid crates are freighter's job to refresh. + if readonly_cache: + if not os.path.exists(crate_path): + warn(f"{label} .crate not in readonly cache: {crate_path}") + return None + if not _crate_file_seems_valid(crate_path): + warn(f"{label} cached .crate appears invalid (readonly; not deleting): {crate_path}") + return None + return crate_path + ensure_directory(os.path.dirname(crate_path)) # Ensure the directory exists download_url = f"{dl_base_url}/{crate_name}/{crate_filename}" - label = _fmt_repo(crate_name, crate_version) def download_once() -> bool: if VERBOSE: @@ -606,6 +638,98 @@ def _git_has_any_commit(repo_path: str) -> bool: ) return res.returncode == 0 +def _sha_is_zero(sha: str) -> bool: + return bool(sha) and set(sha.lower()) <= {"0"} + +def remote_repo_has_commits( + git_base_url: str, + rel: str, + auth_token: str | None, + *, + timeout_s: float = 60.0, +) -> bool: + """True if Mega already has at least one non-zero ref on this path.""" + remote_url = f"{git_base_url.rstrip('/')}/{rel}" + cmd = maybe_wrap_git_with_bearer(["git", "ls-remote", remote_url], auth_token) + try: + res = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout_s) + except subprocess.TimeoutExpired: + warn(f"ls-remote timed out for {rel}") + return False + if res.returncode != 0: + return False + for line in (res.stdout or "").splitlines(): + parts = line.split() + if not parts: + continue + sha = parts[0] + if len(sha) >= 40 and not _sha_is_zero(sha): + return True + return False + +def _is_non_fast_forward_rejection(stdout: str, stderr: str) -> bool: + text = f"{stdout or ''}\n{stderr or ''}".lower() + if "non-fast-forward" in text: + return True + if "fetch first" in text: + return True + if "updates were rejected" in text: + return True + return False + +def git_push_main( + repo_path: str, + *, + auth_token: str | None, + force: bool, + force_with_lease: bool, +) -> str: + """Push main to remote 'mega'. Returns 'ok' | 'exists' | 'fail'. + + 'exists' means the remote rejected a non-fast-forward update — typically + because a prior Job already imported this path. Callers should treat that + as success and record manifest status=ok (unless --force / --force-with-lease). + """ + push_args = ["git", "push", "-u", "mega", "main"] + if force_with_lease: + push_args.insert(2, "--force-with-lease") + elif force: + push_args.insert(2, "--force") + push_cmd = maybe_wrap_git_with_bearer(push_args, auth_token) + try: + result = subprocess.run( + push_cmd, + cwd=repo_path, + capture_output=True, + text=True, + check=False, + ) + except Exception as e: + warn(f"Git push failed: {e}") + return "fail" + if result.returncode == 0: + return "ok" + out = result.stdout or "" + err = result.stderr or "" + if not force and not force_with_lease and _is_non_fast_forward_rejection(out, err): + return "exists" + warn("Git command failed: push rejected") + if VERBOSE: + if out.strip(): + warn(f"stdout: {out.strip()}") + if err.strip(): + warn(f"stderr: {err.strip()}") + return "fail" + +def _cleanup_local_repo(repo_path: str, *, note: str) -> None: + try: + if os.path.exists(repo_path): + shutil.rmtree(repo_path) + if VERBOSE: + info(f"Removed local repo ({note}): {repo_path}") + except Exception as e: + warn(f"Failed to remove local repo {repo_path}: {e}") + def ensure_remote_and_push_existing( repo_path: str, rel: str, @@ -651,31 +775,25 @@ def ensure_remote_and_push_existing( try: with _with_stage(_active_pushes, label): t_push0 = time.monotonic() - push_args = ['git', 'push', '-u', 'mega', 'main'] - if force_with_lease: - push_args.insert(2, '--force-with-lease') - elif force: - push_args.insert(2, '--force') - push_cmd = maybe_wrap_git_with_bearer(push_args, auth_token) - res = run_git_command(repo_path, push_cmd, log_on_error=True) + outcome = git_push_main( + repo_path, + auth_token=auth_token, + force=force, + force_with_lease=force_with_lease, + ) dt_push = time.monotonic() - t_push0 if dt_push >= 5.0: info(f"{label} push finished in {dt_push:.1f}s") finally: push_sema.release() - if res is None: + if outcome == "fail": # Keep repo on disk for troubleshooting _record_push_fail() return False + if outcome == "exists": + info(f"{label} already on mega (non-fast-forward); treating as ok") _record_push_ok() - - # On success, remove local repo directory to save disk space. - try: - shutil.rmtree(repo_path) - if VERBOSE: - info(f"Removed local repo (existing): {repo_path}") - except Exception as e: - warn(f"Failed to remove local repo {repo_path}: {e}") + _cleanup_local_repo(repo_path, note="existing") return True def init_git_repo(repo_path): @@ -829,31 +947,26 @@ def process_crate_version( try: with _with_stage(_active_pushes, label): t_push0 = time.monotonic() - push_args = ['git', 'push', '-u', 'mega', 'main'] - if force_with_lease: - push_args.insert(2, '--force-with-lease') - elif force: - push_args.insert(2, '--force') - push_cmd = maybe_wrap_git_with_bearer(push_args, auth_token) - push_result = run_git_command(repo_path, push_cmd) + outcome = git_push_main( + repo_path, + auth_token=auth_token, + force=force, + force_with_lease=force_with_lease, + ) dt_push = time.monotonic() - t_push0 if dt_push >= 5.0: info(f"{label} push finished in {dt_push:.1f}s") finally: push_sema.release() - if push_result is None: + if outcome == "fail": warn(f"{_fmt_repo(crate_name, version)} push failed") _record_push_fail() return False + if outcome == "exists": + info(f"{label} already on mega (non-fast-forward); treating as ok") _record_push_ok() - # On success, remove local repo directory to save disk space. - try: - shutil.rmtree(repo_path) - if VERBOSE: - info(f"Removed local repo: {repo_path}") - except Exception as e: - warn(f"Failed to remove local repo {repo_path}: {e}") + _cleanup_local_repo(repo_path, note="import") # Optionally drop cached .crate (keep when sharing a host freighter cache). if not keep_crate_cache: try: @@ -1045,11 +1158,17 @@ def scan_and_process_crates( manifest_path: str, reimport_ok: bool, keep_crate_cache: bool = False, + readonly_crate_cache: bool = False, ) -> tuple[int, int, int]: global _progress_jobs, _work_queue info("Scanning crates.io index...") _progress_reset() + if readonly_crate_cache: + # Readonly freighter cache implies never deleting .crate after push. + keep_crate_cache = True + info("Crate cache is readonly: no download/delete under --crates-dir (freighter owns updates).") + stop_evt = threading.Event() hb_thread = None if STATUS_HEARTBEAT: @@ -1106,6 +1225,7 @@ def scan_and_process_crates( f"Config: jobs={_progress_jobs} " f"max_versions_per_crate={max_versions_per_crate} " f"keep_crate_cache={keep_crate_cache} " + f"readonly_crate_cache={readonly_crate_cache} " f"reimport_ok={reimport_ok} dry_run={dry_run}" ) @@ -1120,29 +1240,48 @@ def process_one(crate_name: str, v: str) -> tuple[str, str, str]: rel = mega_third_party_crates_rel_path(crate_name, v) repo_path = os.path.join(git_repos_dir, rel) + label = _fmt_repo(crate_name, v) - # Existing repo path + # Already on Mega (prior Job / NFF leftovers): skip download+push. + # Return "present" → counted as skip, persisted as ok for next runs. + if not dry_run and not reimport_ok and remote_repo_has_commits( + git_base_url, rel, auth_token + ): + if VERBOSE: + info(f"{label} already on mega; skipping") + _cleanup_local_repo(repo_path, note="reconcile") + return ("present", crate_name, v) + + # Existing local workdir: resume push (non-ff → present/ok). if os.path.exists(repo_path) and os.path.exists(os.path.join(repo_path, ".git")): - if repush_existing and not dry_run: - ok_push = ensure_remote_and_push_existing( - repo_path, - rel, - git_base_url, - crate_name=crate_name, - version=v, - commit_signoff=commit_signoff, - auth_token=auth_token, - force=force, - force_with_lease=force_with_lease, - push_sema=push_sema, - ) - return ("ok" if ok_push else "fail", crate_name, v) - else: + if dry_run: if VERBOSE: - info(f"{_fmt_repo(crate_name, v)} exists; skipping") + info(f"{label} exists; skipping") return ("skip", crate_name, v) - - crate_path = check_and_download_crate(crates_dir, crate_name, v, dl_base_url) + ok_push = ensure_remote_and_push_existing( + repo_path, + rel, + git_base_url, + crate_name=crate_name, + version=v, + commit_signoff=commit_signoff, + auth_token=auth_token, + force=force, + force_with_lease=force_with_lease, + push_sema=push_sema, + ) + # ensure_remote_and_push_existing treats NFF as success; map to present + # when we only confirmed remote history (no new objects). Keep "ok" for + # real pushes — both persist as ok via record_result_status. + return ("ok" if ok_push else "fail", crate_name, v) + + crate_path = check_and_download_crate( + crates_dir, + crate_name, + v, + dl_base_url, + readonly_cache=readonly_crate_cache, + ) if crate_path is None: return ("fail", crate_name, v) try: @@ -1168,14 +1307,18 @@ def process_one(crate_name: str, v: str) -> tuple[str, str, str]: def record_result_status(status: str, c_name: str, v: str) -> None: nonlocal succeeded, failed, skipped - _progress_note_result(status) - # Print after progress counters update so sticky footer shows the new totals. - if status == "ok": + # "present" = remote already had the crate; skip work, persist as ok. + persist_status = "ok" if status in ("ok", "present") else status + progress_status = "skip" if status == "present" else status + _progress_note_result(progress_status) + # Counters feed the sticky progress footer (heartbeat ~2s). Per-crate OK + # lines are verbose-only so a full import does not scroll millions of times. + if status == "ok" and VERBOSE: ok(f"{_fmt_repo(c_name, v)} pushed") with lock: if status == "ok": succeeded += 1 - elif status == "skip": + elif status in ("skip", "present"): skipped += 1 else: failed += 1 @@ -1184,7 +1327,7 @@ def record_result_status(status: str, c_name: str, v: str) -> None: rec = { "crate": c_name, "version": v, - "status": status, + "status": persist_status, "remote": f"{git_base_url.rstrip('/')}/{rel}", "last_import_time": datetime.now(timezone.utc).isoformat(), } @@ -1344,8 +1487,8 @@ def main(): p.add_argument( "--status-interval", type=float, - default=10.0, - help="Seconds between status heartbeat prints (default: 10).", + default=2.0, + help="Seconds between status heartbeat / progress refreshes (default: 2).", ) p.add_argument( "--status-sticky", @@ -1397,6 +1540,15 @@ def main(): action="store_true", help="Do not delete downloaded .crate files after a successful push (for shared host caches).", ) + p.add_argument( + "--readonly-crate-cache", + action="store_true", + help=( + "Treat --crates-dir as read-only: never download, create dirs, or delete .crate files. " + "Missing/invalid crates fail; freighter (or another process) must refresh the cache. " + "Implies --keep-crate-cache." + ), + ) args = p.parse_args() global VERBOSE @@ -1406,7 +1558,7 @@ def main(): DOWNLOAD_PROGRESS_INTERVAL_S = float(args.download_progress_interval or 1.0) global STATUS_HEARTBEAT, STATUS_HEARTBEAT_INTERVAL_S STATUS_HEARTBEAT = bool(args.status_heartbeat) - STATUS_HEARTBEAT_INTERVAL_S = float(args.status_interval or 15.0) + STATUS_HEARTBEAT_INTERVAL_S = float(args.status_interval or 2.0) global STATUS_STICKY STATUS_STICKY = bool(args.status_sticky) @@ -1418,7 +1570,13 @@ def main(): crates_dir = str(Path(args.crates_dir).resolve()) git_repos_dir = str(Path(args.workdir).resolve()) - ensure_directory(crates_dir) + readonly_crate_cache = bool(args.readonly_crate_cache) + if readonly_crate_cache: + if not os.path.isdir(crates_dir): + warn(f"Error: --readonly-crate-cache requires existing crates dir: {crates_dir}") + sys.exit(1) + else: + ensure_directory(crates_dir) ensure_directory(git_repos_dir) auth_token = args.token.strip() or None @@ -1450,7 +1608,8 @@ def main(): manifest=manifest, manifest_path=manifest_path, reimport_ok=args.reimport_ok, - keep_crate_cache=bool(args.keep_crate_cache), + keep_crate_cache=bool(args.keep_crate_cache) or readonly_crate_cache, + readonly_crate_cache=readonly_crate_cache, ) # Record end time and calculate duration for the entire process diff --git a/scripts/crates-sync/run_job.py b/scripts/crates-sync/run_job.py index af1a51584..726c04eba 100644 --- a/scripts/crates-sync/run_job.py +++ b/scripts/crates-sync/run_job.py @@ -2,13 +2,17 @@ """K8s Job entrypoint for crates-sync. Waits for mono-engine, bootstraps a bot push token via MEGA_INIT_BOOTSTRAP_SECRET, -optionally refreshes a local crates.io-index checkout, then runs crates-sync.py. +then runs crates-sync.py against a freighter hostPath. + +Freighter owns updates to crates.io-index and the .crate cache. This entrypoint +treats those trees as read-only (no git pull; no download/delete under crates/). +Optional --pull-index is opt-in only. Typical freighter hostPath layout (mounted at --freighter-root): - /crates.io-index -> --index - /crates -> --crates-dir - /mega-crates-work -> --workdir (+ manifest) + /crates.io-index -> --index (read-only) + /crates -> --crates-dir (read-only cache) + /mega-crates-work -> --workdir (+ manifest; writable) """ from __future__ import annotations @@ -157,10 +161,15 @@ def main(argv: list[str] | None = None) -> int: default=0, help="0 = all versions per crate (default for Job).", ) + p.add_argument( + "--pull-index", + action="store_true", + help="git pull the crates.io-index checkout before sync (writes to index; off by default).", + ) p.add_argument( "--no-pull-index", action="store_true", - help="Do not attempt git pull on the index checkout.", + help="Deprecated no-op: index pull is already off by default (freighter owns index updates).", ) p.add_argument( "--wait-timeout", @@ -184,7 +193,10 @@ def main(argv: list[str] | None = None) -> int: raise SystemExit(f"Index directory not found: {index_path}") if not (index_path / "config.json").is_file(): raise SystemExit(f"Index config.json not found under {index_path}") - crates_dir.mkdir(parents=True, exist_ok=True) + if not crates_dir.is_dir(): + raise SystemExit( + f"Crates cache directory not found (readonly freighter layout): {crates_dir}" + ) workdir.mkdir(parents=True, exist_ok=True) base_url = args.base_url.rstrip("/") @@ -192,14 +204,21 @@ def main(argv: list[str] | None = None) -> int: init_secret = resolve_init_bootstrap_secret(args.init_secret) token = bootstrap_init_bot_token(base_url, init_secret) - if not args.no_pull_index: - maybe_pull_index(index_path) + # Freighter owns index/crates updates. Only pull when explicitly requested. + if args.pull_index: + if args.no_pull_index: + print("Warning: --pull-index ignored because --no-pull-index was also set.") + else: + maybe_pull_index(index_path) + elif args.no_pull_index: + print("Index pull skipped (default; --no-pull-index is redundant).") if not CRATES_SYNC_PY.is_file(): raise SystemExit(f"crates-sync.py not found next to run_job.py: {CRATES_SYNC_PY}") cmd = [ sys.executable, + "-u", str(CRATES_SYNC_PY), "--index", str(index_path), @@ -218,12 +237,14 @@ def main(argv: list[str] | None = None) -> int: "--jobs", str(args.jobs), "--keep-crate-cache", + "--readonly-crate-cache", "--status-sticky", ] cmd.extend(extra) printable = [ sys.executable, + "-u", str(CRATES_SYNC_PY), "--index", str(index_path), @@ -242,6 +263,7 @@ def main(argv: list[str] | None = None) -> int: "--jobs", str(args.jobs), "--keep-crate-cache", + "--readonly-crate-cache", "--status-sticky", *extra, ] diff --git "a/scripts/crates-sync/\344\270\212\346\211\213\346\214\207\345\215\227.md" "b/scripts/crates-sync/\344\270\212\346\211\213\346\214\207\345\215\227.md" index f9d1ee74d..2722245f9 100644 --- "a/scripts/crates-sync/\344\270\212\346\211\213\346\214\207\345\215\227.md" +++ "b/scripts/crates-sync/\344\270\212\346\211\213\346\214\207\345\215\227.md" @@ -55,7 +55,7 @@ third-party/rust/crates/// 要点： 1. **导入进程**跑在节点 `storage-server-01` 上（有 freighter 磁盘），不是随便一个 worker。 -2. **index / .crate 缓存**在宿主机 `/opt/data/freighter`，Job 里挂载为 `/freighter`；**不要**在集群里重新 clone 整份 index。 +2. **index / .crate 缓存**在宿主机 `/opt/data/freighter`，Job 里挂载为 `/freighter`。Job **只读**这两棵树（不 `git pull`、不往 `crates/` 下载/删除）；更新由 freighter 负责。可写的只有 `mega-crates-work/`（workdir + manifest）。 3. **git push** 打到集群内的 **mono-engine**；对象最终进 **RustFS**。磁盘不够会导致 push/保存失败。 4. 该节点有污点 `observe-only=true:NoSchedule`；Terraform 已给 Job 配了 toleration，否则 Pod 会一直 Pending。 @@ -76,6 +76,8 @@ third-party/rust/crates/// - **Producer**：全速扫 index，分母（`versions_found`）涨到真实规模（约 **200 万** version）。 - **Workers（`--jobs`）**：同时从队列取任务做下载/推送。 - 已成功写入 manifest 的 `status=ok` **默认跳过**（可断点续跑）。 +- 远端路径上已有历史时，先 `ls-remote` **跳过**下载/推送（进度里算 skip，manifest 写成 `ok`）；要用新提交覆盖需 `--reimport-ok` 且配合 `--force` / `--force-with-lease`。 +- 若仍走到 push 且遇到 non-fast-forward，同样按已存在处理，不再反复 fail。 --- @@ -242,6 +244,7 @@ python3 scripts/crates-sync/crates-sync.py \ | `--jobs N` | 并发 worker 数；同时限制并发 `git push` | | `--max-versions-per-crate N` | 每 crate 只保留最近 N 个版本；`0` = 全部 | | `--keep-crate-cache` | 成功后不删 `.crate`（共享 freighter 缓存时必须开） | +| `--readonly-crate-cache` | **只读** `--crates-dir`：不下载、不删、不建目录；缺包/坏包直接失败（由 freighter 更新） | | `--manifest PATH` | 断点清单；默认 `/crates-import-manifest.jsonl` | | `--reimport-ok` | 强制重导 manifest 里已是 `ok` 的版本 | | `--force` / `--force-with-lease` | 只影响 git push，**不会**绕过 manifest 的 ok 跳过 | @@ -251,18 +254,20 @@ python3 scripts/crates-sync/crates-sync.py \ 默认开启 sticky heartbeat，大致形如： ```text -progress: [##----------------------------] 0.8% 16000/1850000 eta=48.2h +progress: [##----------------------------] 0.8% 16000/1850000 push/s=1.20 eta=48.2h scan: crates=... versions_found=... status=running ... config: jobs=2 queue_depth=... counts: ok=... skip=... fail=... done=... status: downloading=... pushing=... -push: ok_60s=... per_min=... +push: ok_60s=... per_s=1.20 per_min=... ``` 说明： - 扫 index 未完成时，分母会随 `versions_found` **一直涨**（目标约 200 万），不要用早期百分比当「快做完了」。 -- `[OK] xxx pushed` 会刷日志，进度条钉在底部（sticky）。 +- 默认约 **2s** 刷新一次 sticky 进度（可用 `--status-interval` 调整）；**不会**每导入一个版本就刷一行。 +- `push/s` / `per_s` 为近 60 秒成功 push 速率（`ok_60s / 60`）。 +- `[OK] xxx pushed` 仅在 `--verbose` 时打印；平时看底部进度条的 `ok/skip/fail` 计数即可。 ---