hub/delete-project: expand functionality and acutally delete files

Author: haraldschilly

src/packages/backend/files/path-to-files.ts

@@ -0,0 +1,17 @@
+/*
+ *  This file is part of CoCalc: Copyright © 2020 Sagemath, Inc.
+ *  License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
+ */
+
+// This is used to find files on the share server (public_paths) in "next"
+// and also in the hub, for deleting shared files of projects
+
+import { join } from "node:path";
+
+import { projects } from "@cocalc/backend/data";
+
+// Given a project_id/path, return the directory on the file system where
+// that path should be located.
+export function pathToFiles(project_id: string, path: string): string {
+  return join(projects.replace("[project_id]", project_id), path);
+}

src/packages/database/postgres/delete-projects.ts

@@ -7,6 +7,10 @@
 Code related to permanently deleting projects.
 */
 
+import { promises as fs } from "node:fs";
+import { join } from "node:path";
+
+import { pathToFiles } from "@cocalc/backend/files/path-to-files";
 import getLogger from "@cocalc/backend/logger";
 import { newCounter } from "@cocalc/backend/metrics";
 import getPool from "@cocalc/database/pool";
@@ -105,7 +109,7 @@ SELECT project_id
 FROM projects
 WHERE deleted = true
   AND users IS NULL
-  AND state ->> 'state' != 'deleted'
+  AND coalesce(state ->> 'state', '') != 'deleted'
 ORDER BY created ASC
 LIMIT 1000
 `;
@@ -169,10 +173,12 @@ export async function cleanup_old_projects_data(
 
       if (on_prem) {
         L2(`delete all project files`);
-        // TODO: this only works on-prem, and requires the project files to be mounted
+        await deleteProjectFiles(L2, project_id);
 
         L2(`deleting all shared files`);
-        // TODO: do it directly like above, and also get rid of all those shares in the database
+        // this is something like /shared/projects/${project_id}
+        const shared_path = pathToFiles(project_id, "");
+        await fs.rm(shared_path, { recursive: true, force: true });
 
         // for now, on-prem only as well. This gets rid of all sorts of data in tables specific to the given project.
         delRows += await delete_associated_project_data(L2, project_id);
@@ -261,3 +267,24 @@ async function delete_associated_project_data(
 
   return total;
 }
+
+async function deleteProjectFiles(L2, project_id: string) {
+  // TODO: this only works on-prem, and requires the project files to be mounted
+  const projects_root = process.env["MOUNTED_PROJECTS_ROOT"];
+  if (!projects_root) return;
+  const project_dir = join(projects_root, project_id);
+  try {
+    await fs.access(project_dir, fs.constants.F_OK | fs.constants.R_OK);
+    const stats = await fs.lstat(project_dir);
+    if (stats.isDirectory()) {
+      L2(`deleting all files in ${project_dir}`);
+      await fs.rm(project_dir, { recursive: true, force: true });
+    } else {
+      L2(`is not a directory: ${project_dir}`);
+    }
+  } catch (err) {
+    L2(
+      `not deleting project files: either it does not exist or is not accessible`,
+    );
+  }
+}

src/packages/next/lib/share/get-contents.ts

@@ -3,22 +3,24 @@
  *  License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
  */
 
-import pathToFiles from "./path-to-files";
 import { promises as fs } from "fs";
-import { join } from "path";
 import { sortBy } from "lodash";
+import { join } from "path";
+
+import { pathToFiles } from "@cocalc/backend/files/path-to-files";
 import { hasSpecialViewer } from "@cocalc/frontend/file-extensions";
 import { getExtension } from "./util";
 
 const MB: number = 1000000;
+
 const LIMITS = {
   listing: 10000, // directory listing is truncated after this many files
   ipynb: 15 * MB,
   sagews: 10 * MB,
   whiteboard: 5 * MB,
   slides: 5 * MB,
   other: 2 * MB,
-};
+} as const;
 
 export interface FileInfo {
   name: string;
@@ -40,7 +42,7 @@ export interface PathContents {
 
 export default async function getContents(
   project_id: string,
-  path: string
+  path: string,
 ): Promise<PathContents> {
   const fsPath = pathToFiles(project_id, path);
   const obj: PathContents = {};
@@ -72,7 +74,7 @@ export default async function getContents(
 }
 
 async function getDirectoryListing(
-  path: string
+  path: string,
 ): Promise<{ listing: FileInfo[]; truncated?: string }> {
   const listing: FileInfo[] = [];
   let truncated: string | undefined = undefined;

src/packages/next/lib/share/path-to-files.ts

@@ -3,24 +3,17 @@
  *  License: AGPLv3 s.t. "Commons Clause" – see LICENSE.md for details
  */
 
-import { join } from "path";
+import { pathToFiles } from "@cocalc/backend/files/path-to-files";
 import getPool from "@cocalc/database/pool";
-import { projects } from "@cocalc/backend/data";
-
-// Given a project_id/path, return the directory on the file system where
-// that path should be located.
-export default function pathToFiles(project_id: string, path: string): string {
-  return join(projects.replace("[project_id]", project_id), path);
-}
 
 export async function pathFromID(
-  id: string
+  id: string,
 ): Promise<{ projectPath: string; fsPath: string }> {
   // 'infinite' since actually result can't change since id determines the path (it's a reverse sha1 hash computation)
   const pool = getPool("infinite");
   const { rows } = await pool.query(
     "SELECT project_id, path FROM public_paths WHERE id=$1 AND disabled IS NOT TRUE",
-    [id]
+    [id],
   );
   if (rows.length == 0) {
     throw Error(`no such public path: ${id}`);

src/packages/next/lib/share/virtual-hosts.ts

@@ -8,12 +8,13 @@ Support for virtual hosts.
 */
 
 import type { Request, Response } from "express";
+
+import basePath from "@cocalc/backend/base-path";
+import { pathToFiles } from "@cocalc/backend/files/path-to-files";
 import { getLogger } from "@cocalc/backend/logger";
-import pathToFiles from "./path-to-files";
 import isAuthenticated from "./authenticate";
 import getVirtualHostInfo from "./get-vhost-info";
 import { staticHandler } from "./handle-raw";
-import basePath from "@cocalc/backend/base-path";
 
 const logger = getLogger("virtual-hosts");
 
@@ -23,7 +24,7 @@ export default function virtualHostsMiddleware() {
   return async function (
     req: Request,
     res: Response,
-    next: Function
+    next: Function,
   ): Promise<void> {
     // For debugging in cc-in-cc dev, just manually set host to something
     // else and comment this out.  That's the only way, since dev is otherwise
@@ -69,7 +70,7 @@ export default function virtualHostsMiddleware() {
       logger.debug(
         "not authenticated -- denying vhost='%s', path='%s'",
         vhost,
-        path
+        path,
       );
       res.status(403).end();
       return;