mirror of
https://github.com/RayLabsHQ/gitea-mirror.git
synced 2026-10-01 04:51:53 +02:00
feat(export): download repositories and organizations as CSV (#445)
An admin who runs this alone wants to show a team what is being mirrored without handing out a login (#428). GET /api/repositories/export and GET /api/organizations/export return the tracked rows as a CSV file, and an Export CSV button on both pages downloads them. The file is the table without the internal fields: the ids, the metadata sync state and the mirror option overrides. Rows are scoped to the signed in user and ordered by full name and by name. No filters in this pass, the export is always the whole list. toCsv in src/lib/utils/csv.ts does the RFC 4180 work: a header row, CRLF line endings, quoting for commas, quotes and newlines, ISO dates, true and false for booleans, and empty for null.
This commit is contained in:
ARUNAVO RAY
+18
@@ -125,6 +125,24 @@ When several sources are connected, an optional `sourceId` picks which one the o
|
||||
|
||||
`PATCH /api/organizations/:id` with `{ "sourceId": "…" }` re-pins an existing organization to another source, and `{ "sourceId": null }` clears the pin so the organization follows every source's repositories of the same name again. A pin whose source was removed falls back the same way.
|
||||
|
||||
### Export as CSV
|
||||
|
||||
`GET /api/repositories/export` and `GET /api/organizations/export` return everything the account tracks as a CSV file, so the list can be shared with people who do not have a login here (issue #428).
|
||||
|
||||
Both answer `text/csv; charset=utf-8` with a download name such as `gitea-mirror-repositories-2026-09-22.csv`, start with a header row, use CRLF line endings and RFC 4180 quoting, and write dates as ISO strings and booleans as `true` or `false`. Repositories are ordered by full name, organizations by name. The export has no filters: it is every row of the account. Internal fields are left out, that is the ids, the metadata sync state and the mirror option overrides.
|
||||
|
||||
Repository columns: `name`, `fullName`, `url`, `cloneUrl`, `owner`, `organization`, `sourceProvider`, `sourceUrl`, `destinationProvider`, `destinationUrl`, `destinationOrg`, `mirroredLocation`, `visibility`, `isPrivate`, `isForked`, `forkedFrom`, `isStarred`, `isArchived`, `hasLFS`, `hasSubmodules`, `hasIssues`, `language`, `description`, `defaultBranch`, `size`, `status`, `lastMirrored`, `errorMessage`, `importedAt`, `createdAt`, `updatedAt`.
|
||||
|
||||
Organization columns: `name`, `membershipRole`, `isIncluded`, `destinationOrg`, `status`, `repositoryCount`, `publicRepositoryCount`, `privateRepositoryCount`, `forkRepositoryCount`, `lastMirrored`, `errorMessage`, `createdAt`, `updatedAt`.
|
||||
|
||||
```bash
|
||||
curl -sS "https://mirror.example.com/api/repositories/export" \
|
||||
-H "x-api-key: gm_..." \
|
||||
-o repositories.csv
|
||||
```
|
||||
|
||||
The **Export CSV** button on the Repositories and Organizations pages downloads the same files.
|
||||
|
||||
### Reconcile the destination
|
||||
|
||||
`POST /api/cleanup/reconcile`
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { useCallback, useEffect, useState } from "react";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Search, RefreshCw, FlipHorizontal, Filter, LoaderCircle, Trash2 } from "lucide-react";
|
||||
import { Search, RefreshCw, FlipHorizontal, Filter, LoaderCircle, Trash2, Download } from "lucide-react";
|
||||
import type { MirrorJob, Organization } from "@/lib/db/schema";
|
||||
import { OrganizationList } from "./OrganizationsList";
|
||||
import AddOrganizationDialog from "./AddOrganizationDialog";
|
||||
@@ -29,6 +29,7 @@ import { toast } from "sonner";
|
||||
import { invalidateConfigCache, useConfigStatus } from "@/hooks/useConfigStatus";
|
||||
import { useNavigation } from "@/components/layout/MainLayout";
|
||||
import { useLiveRefresh } from "@/hooks/useLiveRefresh";
|
||||
import { withBase } from "@/lib/base-path";
|
||||
import {
|
||||
Drawer,
|
||||
DrawerClose,
|
||||
@@ -568,7 +569,7 @@ export function Organization() {
|
||||
<div className="flex flex-col lg:flex-row items-start lg:items-center gap-2 lg:gap-4 w-full">
|
||||
{/* Mobile: Search bar with filter button */}
|
||||
<div className="flex items-center gap-2 w-full lg:hidden">
|
||||
<div className="relative flex-grow">
|
||||
<div className="relative flex-grow min-w-0">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
|
||||
<input
|
||||
type="text"
|
||||
@@ -786,7 +787,19 @@ export function Organization() {
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
size="icon"
|
||||
title="Export organizations as CSV"
|
||||
className="h-10 w-10 shrink-0"
|
||||
>
|
||||
<a href={withBase("/api/organizations/export")} download>
|
||||
<Download className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="icon"
|
||||
@@ -935,6 +948,19 @@ export function Organization() {
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
{/* CSV export of every tracked organization */}
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
title="Export organizations as CSV"
|
||||
className="h-10 whitespace-nowrap"
|
||||
>
|
||||
<a href={withBase("/api/organizations/export")} download>
|
||||
<Download className="h-4 w-4 xl:mr-2" />
|
||||
<span className="hidden xl:inline">Export CSV</span>
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
onClick={handleMirrorAllOrgs}
|
||||
|
||||
@@ -18,7 +18,7 @@ import {
|
||||
SelectValue,
|
||||
} from "../ui/select";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Search, RefreshCw, FlipHorizontal, RotateCcw, X, Filter, Ban, Check, LoaderCircle, Trash2 } from "lucide-react";
|
||||
import { Search, RefreshCw, FlipHorizontal, RotateCcw, X, Filter, Ban, Check, LoaderCircle, Trash2, Download } from "lucide-react";
|
||||
import type { MirrorRepoRequest, MirrorRepoResponse } from "@/types/mirror";
|
||||
import {
|
||||
Drawer,
|
||||
@@ -1075,7 +1075,7 @@ export default function Repository() {
|
||||
<div className="flex flex-col sm:flex-row items-start sm:items-center gap-2 sm:gap-4 w-full">
|
||||
{/* Mobile: Search bar with filter button */}
|
||||
<div className="flex items-center gap-2 w-full lg:hidden">
|
||||
<div className="relative flex-grow">
|
||||
<div className="relative flex-grow min-w-0">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-muted-foreground" />
|
||||
<input
|
||||
type="text"
|
||||
@@ -1292,7 +1292,19 @@ export default function Repository() {
|
||||
>
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
size="icon"
|
||||
title="Export repositories as CSV"
|
||||
className="h-10 w-10 shrink-0"
|
||||
>
|
||||
<a href={withBase("/api/repositories/export")} download>
|
||||
<Download className="h-4 w-4" />
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
<Button
|
||||
variant="default"
|
||||
size="icon"
|
||||
@@ -1348,6 +1360,19 @@ export default function Repository() {
|
||||
<RefreshCw className="h-4 w-4" />
|
||||
</Button>
|
||||
|
||||
{/* CSV export of every tracked repository */}
|
||||
<Button
|
||||
asChild
|
||||
variant="outline"
|
||||
title="Export repositories as CSV"
|
||||
className="h-10 shrink-0 whitespace-nowrap"
|
||||
>
|
||||
<a href={withBase("/api/repositories/export")} download>
|
||||
<Download className="h-4 w-4 xl:mr-2" />
|
||||
<span className="hidden xl:inline">Export CSV</span>
|
||||
</a>
|
||||
</Button>
|
||||
|
||||
{/* Mirror All action */}
|
||||
<div className="flex items-center gap-2 border-l pl-4">
|
||||
<Button
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
import { describe, test, expect } from "bun:test";
|
||||
import { escapeCsvValue, formatCsvValue, toCsv } from "./csv";
|
||||
|
||||
describe("formatCsvValue", () => {
|
||||
test("empty for null and undefined", () => {
|
||||
expect(formatCsvValue(null)).toBe("");
|
||||
expect(formatCsvValue(undefined)).toBe("");
|
||||
});
|
||||
|
||||
test("dates become ISO strings", () => {
|
||||
expect(formatCsvValue(new Date("2026-09-22T08:30:00.000Z"))).toBe(
|
||||
"2026-09-22T08:30:00.000Z"
|
||||
);
|
||||
});
|
||||
|
||||
test("an invalid date is empty rather than 'Invalid Date'", () => {
|
||||
expect(formatCsvValue(new Date("nope"))).toBe("");
|
||||
});
|
||||
|
||||
test("booleans become true and false", () => {
|
||||
expect(formatCsvValue(true)).toBe("true");
|
||||
expect(formatCsvValue(false)).toBe("false");
|
||||
});
|
||||
|
||||
test("numbers keep their text form, including zero", () => {
|
||||
expect(formatCsvValue(0)).toBe("0");
|
||||
expect(formatCsvValue(42)).toBe("42");
|
||||
});
|
||||
|
||||
test("objects are stringified as JSON", () => {
|
||||
expect(formatCsvValue({ a: 1 })).toBe('{"a":1}');
|
||||
});
|
||||
});
|
||||
|
||||
describe("escapeCsvValue", () => {
|
||||
test("leaves a plain value alone", () => {
|
||||
expect(escapeCsvValue("hello-world")).toBe("hello-world");
|
||||
});
|
||||
|
||||
test("quotes a value with a comma", () => {
|
||||
expect(escapeCsvValue("one, two")).toBe('"one, two"');
|
||||
});
|
||||
|
||||
test("quotes a value with a quote and doubles the quote", () => {
|
||||
expect(escapeCsvValue('say "hi"')).toBe('"say ""hi"""');
|
||||
});
|
||||
|
||||
test("a value that is only quotes doubles every one of them", () => {
|
||||
expect(escapeCsvValue('"""')).toBe('""""""""');
|
||||
});
|
||||
|
||||
test("quotes a value with a line feed", () => {
|
||||
expect(escapeCsvValue("first\nsecond")).toBe('"first\nsecond"');
|
||||
});
|
||||
|
||||
test("quotes a value with a carriage return", () => {
|
||||
expect(escapeCsvValue("first\r\nsecond")).toBe('"first\r\nsecond"');
|
||||
});
|
||||
|
||||
test("keeps leading and trailing spaces without quoting", () => {
|
||||
expect(escapeCsvValue(" padded ")).toBe(" padded ");
|
||||
});
|
||||
});
|
||||
|
||||
describe("toCsv", () => {
|
||||
type Row = {
|
||||
name: string;
|
||||
description: string | null;
|
||||
isPrivate: boolean;
|
||||
size: number;
|
||||
lastMirrored: Date | null;
|
||||
};
|
||||
|
||||
const columns = [
|
||||
"name",
|
||||
"description",
|
||||
"isPrivate",
|
||||
"size",
|
||||
"lastMirrored",
|
||||
] as const;
|
||||
|
||||
test("writes a header row from the column keys", () => {
|
||||
const csv = toCsv<Row>([], columns);
|
||||
expect(csv).toBe("name,description,isPrivate,size,lastMirrored\r\n");
|
||||
});
|
||||
|
||||
test("writes one CRLF terminated line per row", () => {
|
||||
const csv = toCsv<Row>(
|
||||
[
|
||||
{
|
||||
name: "hello-world",
|
||||
description: "A greeting",
|
||||
isPrivate: false,
|
||||
size: 12,
|
||||
lastMirrored: new Date("2026-09-22T08:30:00.000Z"),
|
||||
},
|
||||
{
|
||||
name: "second",
|
||||
description: null,
|
||||
isPrivate: true,
|
||||
size: 0,
|
||||
lastMirrored: null,
|
||||
},
|
||||
],
|
||||
columns
|
||||
);
|
||||
|
||||
expect(csv).toBe(
|
||||
"name,description,isPrivate,size,lastMirrored\r\n" +
|
||||
"hello-world,A greeting,false,12,2026-09-22T08:30:00.000Z\r\n" +
|
||||
"second,,true,0,\r\n"
|
||||
);
|
||||
});
|
||||
|
||||
test("escapes values that hold commas, quotes and newlines", () => {
|
||||
const csv = toCsv<Row>(
|
||||
[
|
||||
{
|
||||
name: "tricky",
|
||||
description: 'Uses "quotes", commas\nand a newline',
|
||||
isPrivate: false,
|
||||
size: 1,
|
||||
lastMirrored: null,
|
||||
},
|
||||
],
|
||||
columns
|
||||
);
|
||||
|
||||
expect(csv.split("\r\n")[1]).toBe(
|
||||
'tricky,"Uses ""quotes"", commas\nand a newline",false,1,'
|
||||
);
|
||||
});
|
||||
|
||||
test("escapes a header that needs quoting too", () => {
|
||||
const csv = toCsv<{ "size, bytes": number }>([{ "size, bytes": 3 }], [
|
||||
"size, bytes",
|
||||
]);
|
||||
expect(csv).toBe('"size, bytes"\r\n3\r\n');
|
||||
});
|
||||
|
||||
test("a missing key on a row is an empty field", () => {
|
||||
const rows = [{ name: "partial" }] as unknown as Row[];
|
||||
const csv = toCsv<Row>(rows, columns);
|
||||
expect(csv.split("\r\n")[1]).toBe("partial,,,,");
|
||||
});
|
||||
|
||||
test("only the listed columns are written, in the listed order", () => {
|
||||
const rows = [
|
||||
{ name: "a", secret: "token", description: "b" },
|
||||
] as unknown as Row[];
|
||||
const csv = toCsv<Row>(rows, ["description", "name"]);
|
||||
expect(csv).toBe("description,name\r\nb,a\r\n");
|
||||
expect(csv).not.toContain("token");
|
||||
});
|
||||
});
|
||||
|
||||
describe("formula injection guard", () => {
|
||||
test("prefixes strings a spreadsheet would run as a formula", () => {
|
||||
expect(formatCsvValue("=1+1")).toBe("'=1+1");
|
||||
expect(formatCsvValue("+SUM(A1)")).toBe("'+SUM(A1)");
|
||||
expect(formatCsvValue("-2+3")).toBe("'-2+3");
|
||||
expect(formatCsvValue("@cmd")).toBe("'@cmd");
|
||||
expect(formatCsvValue("\t=1")).toBe("'\t=1");
|
||||
});
|
||||
|
||||
test("leaves numbers, booleans and ordinary text alone", () => {
|
||||
expect(formatCsvValue(-5)).toBe("-5");
|
||||
expect(formatCsvValue(true)).toBe("true");
|
||||
expect(formatCsvValue("a - b")).toBe("a - b");
|
||||
expect(formatCsvValue("main")).toBe("main");
|
||||
});
|
||||
|
||||
test("still quotes a guarded value that needs quoting", () => {
|
||||
expect(escapeCsvValue('=HYPERLINK("x")')).toBe('"\'=HYPERLINK(""x"")"');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,76 @@
|
||||
/**
|
||||
* Minimal CSV builder used by the repository and organization exports.
|
||||
*
|
||||
* Follows RFC 4180: a header row, CRLF line endings, and fields quoted when
|
||||
* they hold a comma, a double quote, a carriage return or a line feed, with
|
||||
* inner quotes doubled.
|
||||
*/
|
||||
|
||||
/** Fields that need quoting per RFC 4180. */
|
||||
const NEEDS_QUOTING = /[",\r\n]/;
|
||||
|
||||
/**
|
||||
* Text that a spreadsheet would run as a formula. Repository descriptions
|
||||
* and error messages come from third parties, so a value starting with one
|
||||
* of these is prefixed with a single quote, which Excel and Sheets treat as
|
||||
* "this is text". Only strings are affected; numbers stay as they are.
|
||||
*/
|
||||
const FORMULA_PREFIX = /^[=+\-@\t\r]/;
|
||||
|
||||
/**
|
||||
* Turn one value into its CSV text, before any quoting:
|
||||
* null and undefined become empty, dates become ISO strings, booleans become
|
||||
* "true" or "false", and anything else is stringified.
|
||||
*/
|
||||
export function formatCsvValue(value: unknown): string {
|
||||
if (value === null || value === undefined) {
|
||||
return "";
|
||||
}
|
||||
|
||||
if (value instanceof Date) {
|
||||
return Number.isNaN(value.getTime()) ? "" : value.toISOString();
|
||||
}
|
||||
|
||||
if (typeof value === "boolean") {
|
||||
return value ? "true" : "false";
|
||||
}
|
||||
|
||||
if (typeof value === "object") {
|
||||
return JSON.stringify(value) ?? "";
|
||||
}
|
||||
|
||||
if (typeof value === "string" && FORMULA_PREFIX.test(value)) {
|
||||
return `'${value}`;
|
||||
}
|
||||
|
||||
return String(value);
|
||||
}
|
||||
|
||||
/** Format a value and quote it when the content requires it. */
|
||||
export function escapeCsvValue(value: unknown): string {
|
||||
const text = formatCsvValue(value);
|
||||
|
||||
if (!NEEDS_QUOTING.test(text)) {
|
||||
return text;
|
||||
}
|
||||
|
||||
return `"${text.replace(/"/g, '""')}"`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a CSV document from rows and the column keys to export. The keys
|
||||
* double as the header row, so the caller controls both the selection and the
|
||||
* column order.
|
||||
*/
|
||||
export function toCsv<Row>(
|
||||
rows: readonly Row[],
|
||||
columns: readonly (keyof Row & string)[]
|
||||
): string {
|
||||
const lines: string[] = [columns.map(escapeCsvValue).join(",")];
|
||||
|
||||
for (const row of rows) {
|
||||
lines.push(columns.map((column) => escapeCsvValue(row[column])).join(","));
|
||||
}
|
||||
|
||||
return `${lines.join("\r\n")}\r\n`;
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* Route tests for GET /api/organizations/export: the auth guard, the header
|
||||
* row, RFC 4180 escaping of an error message and scoping to the signed in
|
||||
* user (#428).
|
||||
*
|
||||
* Runs against the real schema in an in-memory SQLite database. The route
|
||||
* imports @/lib/db, which is replaced with that database through
|
||||
* mock.module; bun's mock.module is process-wide and leaks into other test
|
||||
* files, so this file registers nothing in the shared process and re-runs
|
||||
* itself in an isolated child (same harness as ./[id]/status.test.ts).
|
||||
*/
|
||||
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as schema from "@/lib/db/schema";
|
||||
|
||||
const CHILD_FLAG = "GM_ORGANIZATIONS_EXPORT_ROUTE_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("organizations CSV export route - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const sqlite = new Database(":memory:");
|
||||
const db = drizzle({ client: sqlite });
|
||||
|
||||
/** Apply every migration in drizzle/ in journal order, the way the migrator would. */
|
||||
function applyMigrations(): void {
|
||||
const folder = join(process.cwd(), "drizzle");
|
||||
const journal = JSON.parse(readFileSync(join(folder, "meta", "_journal.json"), "utf8")) as {
|
||||
entries: { tag: string }[];
|
||||
};
|
||||
for (const entry of journal.entries) {
|
||||
const sql = readFileSync(join(folder, `${entry.tag}.sql`), "utf8");
|
||||
for (const statement of sql.split("--> statement-breakpoint")) {
|
||||
if (statement.trim()) sqlite.run(statement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Flipped by the unauthenticated test. */
|
||||
let authenticatedUserId: string | null = "user-1";
|
||||
|
||||
if (isChild) {
|
||||
applyMigrations();
|
||||
// The seed below only needs the organizations table.
|
||||
sqlite.run("PRAGMA foreign_keys = OFF");
|
||||
|
||||
mock.module("@/lib/auth-guards", () => ({
|
||||
requireAuthenticatedUserId: mock(async () =>
|
||||
authenticatedUserId
|
||||
? { userId: authenticatedUserId }
|
||||
: {
|
||||
response: new Response(
|
||||
JSON.stringify({ success: false, error: "Unauthorized" }),
|
||||
{ status: 401, headers: { "Content-Type": "application/json" } }
|
||||
),
|
||||
}
|
||||
),
|
||||
}));
|
||||
|
||||
mock.module("@/lib/db", () => ({ ...schema, db }));
|
||||
}
|
||||
|
||||
const routeModule = isChild
|
||||
? await import("./export")
|
||||
: ({} as Partial<typeof import("./export")>);
|
||||
const GET = routeModule.GET as NonNullable<typeof routeModule.GET>;
|
||||
const ORGANIZATION_EXPORT_COLUMNS = routeModule.ORGANIZATION_EXPORT_COLUMNS ?? [];
|
||||
|
||||
const EXPECTED_HEADER =
|
||||
"name,membershipRole,isIncluded,destinationOrg,status,repositoryCount," +
|
||||
"publicRepositoryCount,privateRepositoryCount,forkRepositoryCount," +
|
||||
"lastMirrored,errorMessage,createdAt,updatedAt";
|
||||
|
||||
const CREATED_AT = 1_690_000_000;
|
||||
const MIRRORED_AT = 1_700_000_000;
|
||||
|
||||
function iso(seconds: number): string {
|
||||
return new Date(seconds * 1000).toISOString();
|
||||
}
|
||||
|
||||
function get() {
|
||||
return GET({
|
||||
params: {},
|
||||
request: new Request("http://localhost/api/organizations/export"),
|
||||
locals: { session: { userId: "user-1" } },
|
||||
} as any);
|
||||
}
|
||||
|
||||
type OrgSeed = {
|
||||
id: string;
|
||||
userId?: string;
|
||||
name: string;
|
||||
errorMessage?: string | null;
|
||||
};
|
||||
|
||||
function seedOrg({ id, userId = "user-1", name, errorMessage = null }: OrgSeed) {
|
||||
sqlite.run(
|
||||
`INSERT INTO organizations (
|
||||
id, user_id, config_id, name, normalized_name, avatar_url, membership_role,
|
||||
is_included, destination_org, status, repository_count, public_repository_count,
|
||||
private_repository_count, fork_repository_count, last_mirrored, error_message,
|
||||
created_at, updated_at
|
||||
) VALUES (?, ?, 'config-1', ?, ?, 'https://avatars.example.com/a.png', 'admin',
|
||||
1, 'mirrors', 'mirrored', 7, 5,
|
||||
2, 1, ?, ?,
|
||||
?, ?)`,
|
||||
[
|
||||
id,
|
||||
userId,
|
||||
name,
|
||||
name.toLowerCase(),
|
||||
MIRRORED_AT,
|
||||
errorMessage,
|
||||
CREATED_AT,
|
||||
MIRRORED_AT,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
if (!isChild) return;
|
||||
authenticatedUserId = "user-1";
|
||||
sqlite.run("DELETE FROM organizations");
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("GET /api/organizations/export", () => {
|
||||
test("answers 401 when the request is not authenticated", async () => {
|
||||
authenticatedUserId = null;
|
||||
const response = await get();
|
||||
expect(response.status).toBe(401);
|
||||
expect(response.headers.get("Content-Type")).toBe("application/json");
|
||||
expect(await response.json()).toEqual({ success: false, error: "Unauthorized" });
|
||||
});
|
||||
|
||||
test("sends a CSV attachment named after today", async () => {
|
||||
const response = await get();
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Content-Type")).toBe("text/csv; charset=utf-8");
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
expect(response.headers.get("Content-Disposition")).toBe(
|
||||
`attachment; filename="gitea-mirror-organizations-${today}.csv"`
|
||||
);
|
||||
});
|
||||
|
||||
test("writes the header row and no internal fields", async () => {
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
expect(csv.split("\r\n")[0]).toBe(EXPECTED_HEADER);
|
||||
expect(ORGANIZATION_EXPORT_COLUMNS).not.toContain("id" as never);
|
||||
expect(ORGANIZATION_EXPORT_COLUMNS).not.toContain("userId" as never);
|
||||
expect(ORGANIZATION_EXPORT_COLUMNS).not.toContain("configId" as never);
|
||||
expect(ORGANIZATION_EXPORT_COLUMNS).not.toContain("sourceId" as never);
|
||||
expect(ORGANIZATION_EXPORT_COLUMNS).not.toContain("mirrorOverrides" as never);
|
||||
});
|
||||
|
||||
test("writes one row per organization with dates, booleans and counts", async () => {
|
||||
seedOrg({ id: "org-acme", name: "acme" });
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
|
||||
expect(csv.split("\r\n")[1]).toBe(
|
||||
[
|
||||
"acme",
|
||||
"admin",
|
||||
"true",
|
||||
"mirrors",
|
||||
"mirrored",
|
||||
"7",
|
||||
"5",
|
||||
"2",
|
||||
"1",
|
||||
iso(MIRRORED_AT),
|
||||
"",
|
||||
iso(CREATED_AT),
|
||||
iso(MIRRORED_AT),
|
||||
].join(",")
|
||||
);
|
||||
});
|
||||
|
||||
test("quotes an error message with commas, quotes and a newline", async () => {
|
||||
seedOrg({
|
||||
id: "org-acme",
|
||||
name: "acme",
|
||||
errorMessage: 'Gitea said "no", twice\nand gave up',
|
||||
});
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
expect(csv).toContain('"Gitea said ""no"", twice\nand gave up"');
|
||||
});
|
||||
|
||||
test("exports only the signed in user's organizations, ordered by name", async () => {
|
||||
seedOrg({ id: "org-b", name: "beta" });
|
||||
seedOrg({ id: "org-a", name: "Acme" });
|
||||
seedOrg({ id: "org-theirs", userId: "user-2", name: "secret-org" });
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
const lines = csv.split("\r\n").filter((line) => line.length > 0);
|
||||
|
||||
expect(lines).toHaveLength(3);
|
||||
expect(lines[1].startsWith("Acme,")).toBe(true);
|
||||
expect(lines[2].startsWith("beta,")).toBe(true);
|
||||
expect(csv).not.toContain("secret-org");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,59 @@
|
||||
import type { APIRoute } from "astro";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { db, organizations } from "@/lib/db";
|
||||
import { createSecureErrorResponse } from "@/lib/utils";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
import { toCsv } from "@/lib/utils/csv";
|
||||
|
||||
/**
|
||||
* The exported organization columns, in the order they appear in the file. It
|
||||
* is the organizations table without the internal fields: the ids, the avatar
|
||||
* URL and the mirror option overrides (#428).
|
||||
*/
|
||||
const organizationExportSelection = {
|
||||
name: organizations.name,
|
||||
membershipRole: organizations.membershipRole,
|
||||
isIncluded: organizations.isIncluded,
|
||||
destinationOrg: organizations.destinationOrg,
|
||||
status: organizations.status,
|
||||
repositoryCount: organizations.repositoryCount,
|
||||
publicRepositoryCount: organizations.publicRepositoryCount,
|
||||
privateRepositoryCount: organizations.privateRepositoryCount,
|
||||
forkRepositoryCount: organizations.forkRepositoryCount,
|
||||
lastMirrored: organizations.lastMirrored,
|
||||
errorMessage: organizations.errorMessage,
|
||||
createdAt: organizations.createdAt,
|
||||
updatedAt: organizations.updatedAt,
|
||||
};
|
||||
|
||||
export const ORGANIZATION_EXPORT_COLUMNS = Object.keys(
|
||||
organizationExportSelection
|
||||
) as (keyof typeof organizationExportSelection)[];
|
||||
|
||||
export function organizationExportFilename(now: Date): string {
|
||||
return `gitea-mirror-organizations-${now.toISOString().slice(0, 10)}.csv`;
|
||||
}
|
||||
|
||||
export const GET: APIRoute = async ({ request, locals }) => {
|
||||
try {
|
||||
const authResult = await requireAuthenticatedUserId({ request, locals });
|
||||
if ("response" in authResult) return authResult.response;
|
||||
|
||||
const rows = await db
|
||||
.select(organizationExportSelection)
|
||||
.from(organizations)
|
||||
.where(eq(organizations.userId, authResult.userId))
|
||||
.orderBy(sql`${organizations.name} COLLATE NOCASE`);
|
||||
|
||||
return new Response(toCsv(rows, ORGANIZATION_EXPORT_COLUMNS), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${organizationExportFilename(new Date())}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return createSecureErrorResponse(error, "organizations CSV export", 500);
|
||||
}
|
||||
};
|
||||
@@ -0,0 +1,274 @@
|
||||
/**
|
||||
* Route tests for GET /api/repositories/export: the auth guard, the header
|
||||
* row, RFC 4180 escaping of a description and scoping to the signed in user
|
||||
* (#428).
|
||||
*
|
||||
* Runs against the real schema in an in-memory SQLite database. The route
|
||||
* imports @/lib/db, which is replaced with that database through
|
||||
* mock.module; bun's mock.module is process-wide and leaks into other test
|
||||
* files, so this file registers nothing in the shared process and re-runs
|
||||
* itself in an isolated child (same harness as ../organizations/[id]/status.test.ts).
|
||||
*/
|
||||
import { describe, test, expect, mock, beforeEach } from "bun:test";
|
||||
import { Database } from "bun:sqlite";
|
||||
import { drizzle } from "drizzle-orm/bun-sqlite";
|
||||
import { readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import * as schema from "@/lib/db/schema";
|
||||
|
||||
const CHILD_FLAG = "GM_REPOSITORIES_EXPORT_ROUTE_ISOLATED";
|
||||
const isChild = !!process.env[CHILD_FLAG];
|
||||
|
||||
if (!isChild) {
|
||||
test("repositories CSV export route - isolated child suite", () => {
|
||||
const res = Bun.spawnSync({
|
||||
cmd: [process.execPath, "test", import.meta.path],
|
||||
env: { ...process.env, [CHILD_FLAG]: "1" },
|
||||
stdout: "pipe",
|
||||
stderr: "pipe",
|
||||
});
|
||||
if (res.exitCode !== 0) {
|
||||
console.error(res.stdout.toString());
|
||||
console.error(res.stderr.toString());
|
||||
}
|
||||
expect(res.exitCode).toBe(0);
|
||||
}, 60_000);
|
||||
}
|
||||
|
||||
const sqlite = new Database(":memory:");
|
||||
const db = drizzle({ client: sqlite });
|
||||
|
||||
/** Apply every migration in drizzle/ in journal order, the way the migrator would. */
|
||||
function applyMigrations(): void {
|
||||
const folder = join(process.cwd(), "drizzle");
|
||||
const journal = JSON.parse(readFileSync(join(folder, "meta", "_journal.json"), "utf8")) as {
|
||||
entries: { tag: string }[];
|
||||
};
|
||||
for (const entry of journal.entries) {
|
||||
const sql = readFileSync(join(folder, `${entry.tag}.sql`), "utf8");
|
||||
for (const statement of sql.split("--> statement-breakpoint")) {
|
||||
if (statement.trim()) sqlite.run(statement);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Flipped by the unauthenticated test. */
|
||||
let authenticatedUserId: string | null = "user-1";
|
||||
|
||||
if (isChild) {
|
||||
applyMigrations();
|
||||
// The seed below only needs the repositories table.
|
||||
sqlite.run("PRAGMA foreign_keys = OFF");
|
||||
|
||||
mock.module("@/lib/auth-guards", () => ({
|
||||
requireAuthenticatedUserId: mock(async () =>
|
||||
authenticatedUserId
|
||||
? { userId: authenticatedUserId }
|
||||
: {
|
||||
response: new Response(
|
||||
JSON.stringify({ success: false, error: "Unauthorized" }),
|
||||
{ status: 401, headers: { "Content-Type": "application/json" } }
|
||||
),
|
||||
}
|
||||
),
|
||||
}));
|
||||
|
||||
mock.module("@/lib/db", () => ({ ...schema, db }));
|
||||
}
|
||||
|
||||
const routeModule = isChild
|
||||
? await import("./export")
|
||||
: ({} as Partial<typeof import("./export")>);
|
||||
const GET = routeModule.GET as NonNullable<typeof routeModule.GET>;
|
||||
const REPOSITORY_EXPORT_COLUMNS = routeModule.REPOSITORY_EXPORT_COLUMNS ?? [];
|
||||
|
||||
const EXPECTED_HEADER =
|
||||
"name,fullName,url,cloneUrl,owner,organization,sourceProvider,sourceUrl," +
|
||||
"destinationProvider,destinationUrl,destinationOrg,mirroredLocation,visibility," +
|
||||
"isPrivate,isForked,forkedFrom,isStarred,isArchived,hasLFS,hasSubmodules,hasIssues," +
|
||||
"language,description,defaultBranch,size,status,lastMirrored,errorMessage," +
|
||||
"importedAt,createdAt,updatedAt";
|
||||
|
||||
const IMPORTED_AT = 1_690_000_000;
|
||||
const MIRRORED_AT = 1_700_000_000;
|
||||
|
||||
function iso(seconds: number): string {
|
||||
return new Date(seconds * 1000).toISOString();
|
||||
}
|
||||
|
||||
function get() {
|
||||
return GET({
|
||||
params: {},
|
||||
request: new Request("http://localhost/api/repositories/export"),
|
||||
locals: { session: { userId: "user-1" } },
|
||||
} as any);
|
||||
}
|
||||
|
||||
type RepoSeed = {
|
||||
id: string;
|
||||
userId?: string;
|
||||
name: string;
|
||||
fullName: string;
|
||||
organization?: string | null;
|
||||
description?: string | null;
|
||||
};
|
||||
|
||||
function seedRepo({
|
||||
id,
|
||||
userId = "user-1",
|
||||
name,
|
||||
fullName,
|
||||
organization = null,
|
||||
description = null,
|
||||
}: RepoSeed) {
|
||||
const owner = fullName.split("/")[0];
|
||||
sqlite.run(
|
||||
`INSERT INTO repositories (
|
||||
id, user_id, config_id, name, full_name, normalized_full_name, url, clone_url,
|
||||
source_provider, source_url, destination_provider, destination_url, destination_org,
|
||||
owner, organization, mirrored_location, visibility, is_private, is_fork, forked_from,
|
||||
is_starred, is_archived, has_lfs, has_issues, language, description, default_branch,
|
||||
size, status, last_mirrored, error_message, imported_at, created_at, updated_at
|
||||
) VALUES (?, ?, 'config-1', ?, ?, ?, ?, ?,
|
||||
'github', 'https://github.com', 'gitea', 'https://gitea.example.com', 'mirrors',
|
||||
?, ?, ?, 'public', 0, 0, NULL,
|
||||
1, 0, 0, 1, 'TypeScript', ?, 'main',
|
||||
128, 'mirrored', ?, NULL, ?, ?, ?)`,
|
||||
[
|
||||
id,
|
||||
userId,
|
||||
name,
|
||||
fullName,
|
||||
fullName.toLowerCase(),
|
||||
`https://github.com/${fullName}`,
|
||||
`https://github.com/${fullName}.git`,
|
||||
owner,
|
||||
organization,
|
||||
`mirrors/${name}`,
|
||||
description,
|
||||
MIRRORED_AT,
|
||||
IMPORTED_AT,
|
||||
IMPORTED_AT,
|
||||
MIRRORED_AT,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
beforeEach(() => {
|
||||
if (!isChild) return;
|
||||
authenticatedUserId = "user-1";
|
||||
sqlite.run("DELETE FROM repositories");
|
||||
});
|
||||
|
||||
describe.skipIf(!isChild)("GET /api/repositories/export", () => {
|
||||
test("answers 401 when the request is not authenticated", async () => {
|
||||
authenticatedUserId = null;
|
||||
const response = await get();
|
||||
expect(response.status).toBe(401);
|
||||
expect(response.headers.get("Content-Type")).toBe("application/json");
|
||||
expect(await response.json()).toEqual({ success: false, error: "Unauthorized" });
|
||||
});
|
||||
|
||||
test("sends a CSV attachment named after today", async () => {
|
||||
const response = await get();
|
||||
expect(response.status).toBe(200);
|
||||
expect(response.headers.get("Content-Type")).toBe("text/csv; charset=utf-8");
|
||||
const today = new Date().toISOString().slice(0, 10);
|
||||
expect(response.headers.get("Content-Disposition")).toBe(
|
||||
`attachment; filename="gitea-mirror-repositories-${today}.csv"`
|
||||
);
|
||||
});
|
||||
|
||||
test("writes the header row and no internal fields", async () => {
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
expect(csv.split("\r\n")[0]).toBe(EXPECTED_HEADER);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("id" as never);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("userId" as never);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("configId" as never);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("sourceId" as never);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("metadata" as never);
|
||||
expect(REPOSITORY_EXPORT_COLUMNS).not.toContain("mirrorOverrides" as never);
|
||||
});
|
||||
|
||||
test("writes one row per repository with dates, booleans and blanks", async () => {
|
||||
seedRepo({
|
||||
id: "repo-1",
|
||||
name: "hello-world",
|
||||
fullName: "octocat/hello-world",
|
||||
description: "A greeting",
|
||||
});
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
|
||||
expect(csv.split("\r\n")[1]).toBe(
|
||||
[
|
||||
"hello-world",
|
||||
"octocat/hello-world",
|
||||
"https://github.com/octocat/hello-world",
|
||||
"https://github.com/octocat/hello-world.git",
|
||||
"octocat",
|
||||
"",
|
||||
"github",
|
||||
"https://github.com",
|
||||
"gitea",
|
||||
"https://gitea.example.com",
|
||||
"mirrors",
|
||||
"mirrors/hello-world",
|
||||
"public",
|
||||
"false",
|
||||
"false",
|
||||
"",
|
||||
"true",
|
||||
"false",
|
||||
"false",
|
||||
"false",
|
||||
"true",
|
||||
"TypeScript",
|
||||
"A greeting",
|
||||
"main",
|
||||
"128",
|
||||
"mirrored",
|
||||
iso(MIRRORED_AT),
|
||||
"",
|
||||
iso(IMPORTED_AT),
|
||||
iso(IMPORTED_AT),
|
||||
iso(MIRRORED_AT),
|
||||
].join(",")
|
||||
);
|
||||
});
|
||||
|
||||
test("quotes a description with commas, quotes and a newline", async () => {
|
||||
seedRepo({
|
||||
id: "repo-1",
|
||||
name: "tricky",
|
||||
fullName: "octocat/tricky",
|
||||
description: 'Uses "quotes", commas\nand a newline',
|
||||
});
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
expect(csv).toContain('"Uses ""quotes"", commas\nand a newline"');
|
||||
});
|
||||
|
||||
test("exports only the signed in user's repositories, ordered by full name", async () => {
|
||||
seedRepo({ id: "repo-b", name: "beta", fullName: "octocat/beta" });
|
||||
seedRepo({ id: "repo-a", name: "Alpha", fullName: "octocat/Alpha" });
|
||||
seedRepo({
|
||||
id: "repo-theirs",
|
||||
userId: "user-2",
|
||||
name: "secret",
|
||||
fullName: "someone/secret",
|
||||
});
|
||||
|
||||
const response = await get();
|
||||
const csv = await response.text();
|
||||
const lines = csv.split("\r\n").filter((line) => line.length > 0);
|
||||
|
||||
expect(lines).toHaveLength(3);
|
||||
expect(lines[1].startsWith("Alpha,octocat/Alpha,")).toBe(true);
|
||||
expect(lines[2].startsWith("beta,octocat/beta,")).toBe(true);
|
||||
expect(csv).not.toContain("someone/secret");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
import type { APIRoute } from "astro";
|
||||
import { eq, sql } from "drizzle-orm";
|
||||
import { db, repositories } from "@/lib/db";
|
||||
import { createSecureErrorResponse } from "@/lib/utils";
|
||||
import { requireAuthenticatedUserId } from "@/lib/auth-guards";
|
||||
import { toCsv } from "@/lib/utils/csv";
|
||||
|
||||
/**
|
||||
* The exported repository columns, in the order they appear in the file. It
|
||||
* is the repositories table without the internal fields: the ids, the
|
||||
* metadata sync state and the mirror option overrides (#428).
|
||||
*/
|
||||
const repositoryExportSelection = {
|
||||
name: repositories.name,
|
||||
fullName: repositories.fullName,
|
||||
url: repositories.url,
|
||||
cloneUrl: repositories.cloneUrl,
|
||||
owner: repositories.owner,
|
||||
organization: repositories.organization,
|
||||
sourceProvider: repositories.sourceProvider,
|
||||
sourceUrl: repositories.sourceUrl,
|
||||
destinationProvider: repositories.destinationProvider,
|
||||
destinationUrl: repositories.destinationUrl,
|
||||
destinationOrg: repositories.destinationOrg,
|
||||
mirroredLocation: repositories.mirroredLocation,
|
||||
visibility: repositories.visibility,
|
||||
isPrivate: repositories.isPrivate,
|
||||
isForked: repositories.isForked,
|
||||
forkedFrom: repositories.forkedFrom,
|
||||
isStarred: repositories.isStarred,
|
||||
isArchived: repositories.isArchived,
|
||||
hasLFS: repositories.hasLFS,
|
||||
hasSubmodules: repositories.hasSubmodules,
|
||||
hasIssues: repositories.hasIssues,
|
||||
language: repositories.language,
|
||||
description: repositories.description,
|
||||
defaultBranch: repositories.defaultBranch,
|
||||
size: repositories.size,
|
||||
status: repositories.status,
|
||||
lastMirrored: repositories.lastMirrored,
|
||||
errorMessage: repositories.errorMessage,
|
||||
importedAt: repositories.importedAt,
|
||||
createdAt: repositories.createdAt,
|
||||
updatedAt: repositories.updatedAt,
|
||||
};
|
||||
|
||||
export const REPOSITORY_EXPORT_COLUMNS = Object.keys(
|
||||
repositoryExportSelection
|
||||
) as (keyof typeof repositoryExportSelection)[];
|
||||
|
||||
export function repositoryExportFilename(now: Date): string {
|
||||
return `gitea-mirror-repositories-${now.toISOString().slice(0, 10)}.csv`;
|
||||
}
|
||||
|
||||
export const GET: APIRoute = async ({ request, locals }) => {
|
||||
try {
|
||||
const authResult = await requireAuthenticatedUserId({ request, locals });
|
||||
if ("response" in authResult) return authResult.response;
|
||||
|
||||
const rows = await db
|
||||
.select(repositoryExportSelection)
|
||||
.from(repositories)
|
||||
.where(eq(repositories.userId, authResult.userId))
|
||||
.orderBy(sql`${repositories.fullName} COLLATE NOCASE`);
|
||||
|
||||
return new Response(toCsv(rows, REPOSITORY_EXPORT_COLUMNS), {
|
||||
status: 200,
|
||||
headers: {
|
||||
"Content-Type": "text/csv; charset=utf-8",
|
||||
"Content-Disposition": `attachment; filename="${repositoryExportFilename(new Date())}"`,
|
||||
"Cache-Control": "no-store",
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
return createSecureErrorResponse(error, "repositories CSV export", 500);
|
||||
}
|
||||
};
|
||||
Reference in New Issue
Block a user