diff --git a/handwritten/storage/src/file.ts b/handwritten/storage/src/file.ts index 5c51e63dfb0..28a2ec57f47 100644 --- a/handwritten/storage/src/file.ts +++ b/handwritten/storage/src/file.ts @@ -396,6 +396,22 @@ export interface CopyCallback { export type DownloadResponse = [Buffer]; +export type DownloadResponseWithStatus = [Buffer] & { + skipped?: boolean; + reason?: SkipReason; + fileName?: string; + localPath?: string; + message?: string; + error?: Error; +}; + +export enum SkipReason { + PATH_TRAVERSAL = 'PATH_TRAVERSAL', + ILLEGAL_CHARACTER = 'ILLEGAL_CHARACTER', + ALREADY_EXISTS = 'ALREADY_EXISTS', + DOWNLOAD_ERROR = 'DOWNLOAD_ERROR', +} + export type DownloadCallback = ( err: RequestError | null, contents: Buffer, diff --git a/handwritten/storage/src/transfer-manager.ts b/handwritten/storage/src/transfer-manager.ts index be34c76f08e..326573e475f 100644 --- a/handwritten/storage/src/transfer-manager.ts +++ b/handwritten/storage/src/transfer-manager.ts @@ -18,9 +18,11 @@ import {Bucket, UploadOptions, UploadResponse} from './bucket.js'; import { DownloadOptions, DownloadResponse, + DownloadResponseWithStatus, File, FileExceptionMessages, RequestError, + SkipReason, } from './file.js'; import pLimit from 'p-limit'; import * as path from 'path'; @@ -570,9 +572,13 @@ export class TransferManager { const limit = pLimit( options.concurrencyLimit || DEFAULT_PARALLEL_DOWNLOAD_LIMIT, ); - const promises: Promise[] = []; + const promises: Promise[] = []; let files: File[] = []; + const baseDestination = path.resolve( + options.passthroughOptions?.destination || '.' + ); + if (!Array.isArray(filesOrFolder)) { const directoryFiles = await this.bucket.getFiles({ prefix: filesOrFolder, @@ -592,45 +598,96 @@ export class TransferManager { : EMPTY_REGEX; const regex = new RegExp(stripRegexString, 'g'); - for (const file of files) { - const passThroughOptionsCopy = { - ...options.passthroughOptions, - [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY, - }; + const finalResults: DownloadResponseWithStatus[] = new Array(files.length); - if (options.prefix || passThroughOptionsCopy.destination) { - passThroughOptionsCopy.destination = path.join( - options.prefix || '', - passThroughOptionsCopy.destination || '', - file.name, - ); + for (let i = 0; i < files.length; i++) { + const file = files[i]; + + const hasIllegalDrive = /^[a-zA-Z]:/.test(file.name); + if (hasIllegalDrive) { + const skippedResult = [Buffer.alloc(0)] as DownloadResponseWithStatus; + skippedResult.skipped = true; + skippedResult.reason = SkipReason.ILLEGAL_CHARACTER; + skippedResult.fileName = file.name; + finalResults[i] = skippedResult; + continue; } - if (options.stripPrefix) { - passThroughOptionsCopy.destination = file.name.replace(regex, ''); + + const normalizedGcsName = file.name.replace(/^[\\/]+/, ''); + + const dest = options.stripPrefix + ? normalizedGcsName.replace(regex, '') + : normalizedGcsName; + + const resolvedPath = path.resolve(baseDestination, dest); + const relativeFromBase = path.relative(baseDestination, resolvedPath); + + const isOutside = + path.isAbsolute(relativeFromBase) || + relativeFromBase.split(/[\\/]/).includes('..'); + + if (isOutside) { + const skippedResult = [Buffer.alloc(0)] as DownloadResponseWithStatus; + skippedResult.skipped = true; + skippedResult.reason = SkipReason.PATH_TRAVERSAL; + skippedResult.fileName = file.name; + skippedResult.localPath = resolvedPath; + finalResults[i] = skippedResult; + continue; } - if ( - options.skipIfExists && - existsSync(passThroughOptionsCopy.destination || '') - ) { + + if (options.skipIfExists && existsSync(resolvedPath)) { continue; } promises.push( limit(async () => { - const destination = passThroughOptionsCopy.destination; - if (destination && destination.endsWith(path.sep)) { - await fsp.mkdir(destination, {recursive: true}); - return Promise.resolve([ - Buffer.alloc(0), - ]) as Promise; + const passThroughOptionsCopy = { + ...options.passthroughOptions, + destination: resolvedPath, + [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.DOWNLOAD_MANY, + }; + + try { + const destination = passThroughOptionsCopy.destination!; + + if (destination.endsWith(path.sep) || destination.endsWith('/')) { + await fsp.mkdir(destination, {recursive: true}); + const dirResp = [Buffer.alloc(0)] as DownloadResponseWithStatus; + dirResp.skipped = false; + dirResp.fileName = file.name; + dirResp.localPath = destination; + finalResults[i] = dirResp; + return; + } + + await fsp.mkdir(path.dirname(destination), {recursive: true}); + + const resp = (await file.download( + passThroughOptionsCopy + )) as DownloadResponseWithStatus; + + finalResults[i] = { + ...resp, + skipped: false, + fileName: file.name, + localPath: destination, + } as DownloadResponse; + } catch (err) { + const errorResp = [Buffer.alloc(0)] as DownloadResponseWithStatus; + errorResp.skipped = true; + errorResp.reason = SkipReason.DOWNLOAD_ERROR; + errorResp.fileName = file.name; + errorResp.localPath = resolvedPath; + errorResp.error = err as Error; + finalResults[i] = errorResp; } - - return file.download(passThroughOptionsCopy); - }), + }) ); } - return Promise.all(promises); + await Promise.all(promises); + return finalResults; } /** diff --git a/handwritten/storage/test/transfer-manager.ts b/handwritten/storage/test/transfer-manager.ts index 2582782fa7a..7433f05462e 100644 --- a/handwritten/storage/test/transfer-manager.ts +++ b/handwritten/storage/test/transfer-manager.ts @@ -32,6 +32,7 @@ import { DownloadManyFilesOptions, } from '../src/index.js'; import assert from 'assert'; +import {describe, it, beforeEach, before, afterEach, after} from 'mocha'; import * as path from 'path'; import {GaxiosOptions, GaxiosResponse} from 'gaxios'; import {GCCL_GCS_CMD_KEY} from '../src/nodejs-common/util.js'; @@ -39,8 +40,8 @@ import {AuthClient, GoogleAuth} from 'google-auth-library'; import {tmpdir} from 'os'; import fs from 'fs'; import {promises as fsp, Stats} from 'fs'; - import * as sinon from 'sinon'; +import {DownloadResponseWithStatus, SkipReason} from '../src/file.js'; describe('Transfer Manager', () => { const BUCKET_NAME = 'test-bucket'; @@ -324,7 +325,7 @@ describe('Transfer Manager', () => { const file = 'first.txt'; const filesOrFolder = [folder, path.join(folder, file)]; const expectedFilePath = path.join(prefix, folder, file); - const expectedDir = path.join(prefix, folder); + const expectedDir = path.resolve(prefix, folder); const mkdirSpy = sandbox.spy(fsp, 'mkdir'); const download = (optionsOrCb?: DownloadOptions | DownloadCallback) => { if (typeof optionsOrCb === 'function') { @@ -341,15 +342,220 @@ describe('Transfer Manager', () => { return file; }); await transferManager.downloadManyFiles(filesOrFolder, { - prefix: prefix, + passthroughOptions: {destination: prefix}, }); assert.strictEqual( - mkdirSpy.calledOnceWith(expectedDir, { - recursive: true, - }), + mkdirSpy.calledWith(expectedDir, {recursive: true}), true ); }); + + it('skips files that attempt path traversal via dot-segments (../) and returns them in skippedFiles', async () => { + const destination = 'download-directory'; + const maliciousFilename = '../../etc/passwd'; + const validFilename = 'valid.txt'; + + const maliciousFile = new File(bucket, maliciousFilename); + const validFile = new File(bucket, validFilename); + + const downloadStub = sandbox + .stub(validFile, 'download') + .resolves([Buffer.alloc(0)]); + const maliciousDownloadStub = sandbox.stub(maliciousFile, 'download'); + + const result = (await transferManager.downloadManyFiles( + [maliciousFile, validFile], + {passthroughOptions: {destination: destination}} + )) as DownloadResponseWithStatus[]; + + assert.strictEqual(maliciousDownloadStub.called, false); + assert.strictEqual(downloadStub.calledOnce, true); + + const skipped = result.find(r => r.fileName === maliciousFilename); + assert.ok(skipped); + assert.strictEqual(skipped!.skipped, true); + assert.strictEqual(skipped!.reason, SkipReason.PATH_TRAVERSAL); + }); + + it('allows files with relative segments that resolve within the target directory', async () => { + const destination = 'safe-directory'; + const filename = './subdir/../subdir/file.txt'; + const file = new File(bucket, filename); + + const downloadStub = sandbox + .stub(file, 'download') + .resolves([Buffer.alloc(0)]); + + await transferManager.downloadManyFiles([file], { + passthroughOptions: {destination: destination}, + }); + + assert.strictEqual(downloadStub.calledOnce, true); + }); + + it('prevents traversal when no prefix is provided', async () => { + const maliciousFilename = '../../../traversal.txt'; + const file = new File(bucket, maliciousFilename); + const downloadStub = sandbox.stub(file, 'download'); + + const result = (await transferManager.downloadManyFiles([ + file, + ])) as DownloadResponseWithStatus[]; + + assert.strictEqual(downloadStub.called, false); + assert.strictEqual(result[0].skipped, true); + assert.strictEqual(result[0].reason, SkipReason.PATH_TRAVERSAL); + }); + + it('jails absolute-looking paths with nested segments into the target directory', async () => { + const destination = './downloads'; + const filename = '/tmp/shady.txt'; + const file = new File(bucket, filename); + const expectedDestination = path.resolve( + destination, + filename.replace(/^\/+/, '') + ); + + const downloadStub = sandbox + .stub(file, 'download') + .resolves([Buffer.alloc(0)]); + + const result = (await transferManager.downloadManyFiles([file], { + passthroughOptions: {destination: destination}, + })) as DownloadResponseWithStatus[]; + + assert.strictEqual(downloadStub.called, true); + const options = downloadStub.firstCall.args[0] as DownloadOptions; + assert.strictEqual(options.destination, expectedDestination); + + assert.strictEqual(result.length, 1); + assert.strictEqual(result[0].skipped, false); + }); + + it('jails absolute-looking Unix paths (e.g. /etc/passwd) into the target directory instead of skipping', async () => { + const destination = 'downloads'; + const filename = '/etc/passwd'; + const expectedDestination = path.resolve( + destination, + filename.replace(/^\/+/, '') + ); + + const file = new File(bucket, filename); + const downloadStub = sandbox + .stub(file, 'download') + .resolves([Buffer.alloc(0)]); + + const result = (await transferManager.downloadManyFiles([file], { + passthroughOptions: {destination: destination}, + })) as DownloadResponseWithStatus[]; + + assert.strictEqual(downloadStub.calledOnce, true); + const options = downloadStub.firstCall.args[0] as DownloadOptions; + assert.strictEqual(options.destination, expectedDestination); + assert.strictEqual(result[0].skipped, false); + }); + + it('correctly handles stripPrefix and verifies the resulting path is still safe', async () => { + const options = { + stripPrefix: 'secret/', + prefix: 'local-folder', + }; + const filename = 'secret/../../escape.txt'; + const file = new File(bucket, filename); + + const downloadStub = sandbox.stub(file, 'download'); + + const result = (await transferManager.downloadManyFiles( + [file], + options + )) as DownloadResponseWithStatus[]; + + assert.strictEqual(downloadStub.called, false); + assert.strictEqual(result[0].skipped, true); + assert.strictEqual(result[0].reason, SkipReason.PATH_TRAVERSAL); + }); + + it('should skip files containing Windows volume separators (:) to prevent drive-injection attacks', async () => { + const destination = 'C:\\local\\target'; + const maliciousFile = new File(bucket, 'C:\\system\\win32'); + + const result = (await transferManager.downloadManyFiles([maliciousFile], { + passthroughOptions: {destination: destination}, + })) as DownloadResponseWithStatus[]; + + assert.strictEqual(result.length, 1); + + const response = result[0]; + assert.strictEqual(response.skipped, true); + assert.strictEqual(response.reason, SkipReason.ILLEGAL_CHARACTER); + assert.strictEqual(response.fileName, 'C:\\system\\win32'); + }); + + it('should account for every input file (Parity Check)', async () => { + const destination = '/local/target'; + const fileNames = [ + 'data/file.txt', // Normal (Download) + 'data/../sibling.txt', // Internal Traversal (Download) + '../escape.txt', // External Traversal (Skip - Path Traversal '..') + '/etc/passwd', // Leading Slash (Download) + '/local/usr/a.txt', // Path matches prefix (Download) + 'dir/./file.txt', // Dot segment (Download) + 'windows\\file.txt', // Windows separator (Download) + 'data\\..\\sibling.txt', // Windows traversal (Download) + '..\\escape.txt', // Windows escape (Skip - Path Traversal '..') + 'C:\\system\\win32', // Windows Drive (Skip - Illegal Char ':') + 'C:\\local\\target\\a.txt', // Windows Absolute (Skip - Illegal Char ':') + '..temp.txt', // Leading dots in filename (Download - Not a traversal) + 'test-2026:01:01.txt', // GCS Timestamps (Download - Colon is middle, not drive) + '\\a\\b\\c.txt', // Leading backslash (Should stay in base) + '/abs/path/file.txt', // Leading forward slash (Should stay in base) + '\\\\network\\share', // UNC-style leading double backslash + '//multiple//slashes', // Multiple leading forward slashes + ]; + + const files = fileNames.map(name => bucket.file(name)); + + sandbox.stub(File.prototype, 'download').resolves([Buffer.alloc(0)]); + sandbox.stub(fsp, 'mkdir').resolves(); + + const result = (await transferManager.downloadManyFiles(files, { + passthroughOptions: {destination: destination}, + })) as DownloadResponseWithStatus[]; + + assert.strictEqual( + result.length, + fileNames.length, + `Parity Failure: Processed ${result.length} files but input had ${fileNames.length}` + ); + + const downloads = result.filter(r => !r.skipped); + const skips = result.filter(r => r.skipped); + + const expectedDownloads = 12; + const expectedSkips = 5; + + assert.strictEqual( + downloads.length, + expectedDownloads, + `Expected ${expectedDownloads} downloads but got ${downloads.length}` + ); + + assert.strictEqual( + skips.length, + expectedSkips, + `Expected ${expectedSkips} skips but got ${skips.length}` + ); + + const traversalSkips = skips.filter( + f => f.reason === SkipReason.PATH_TRAVERSAL + ); + assert.strictEqual(traversalSkips.length, 3); + + const illegalCharSkips = skips.filter( + f => f.reason === SkipReason.ILLEGAL_CHARACTER + ); + assert.strictEqual(illegalCharSkips.length, 2); + }); }); describe('downloadFileInChunks', () => {