From 35960c5de68885812e31dfe93dd89298ea65831d Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Sat, 13 Aug 2022 18:01:07 +0000 Subject: [PATCH 1/8] Use zstd short flag version for win runners. Update comments to reflect the same. Fix --d to -d in comments. --- src/internal/tar.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/internal/tar.ts b/src/internal/tar.ts index 1f63da5..7c45a93 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -61,14 +61,17 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - // --d: Decompress. + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: + if (process.platform === 'win32') return ['--use-compress-program=zstd -d --long=30'] return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: + if (process.platform === 'win32') return ['--use-compress-program=zstd -d'] return ['--use-compress-program', 'unzstd'] default: return ['-z'] @@ -100,14 +103,17 @@ export async function createTar( const workingDirectory = getWorkingDirectory() // -T#: Compress using # working thread. If # is 0, attempt to detect and use the number of physical CPU cores. + // zstdmt is equivalent to 'zstd -T0' // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. // Long range mode is added to zstd in v1.3.2 release, so we will not use --long in older version of zstd. function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: + if (process.platform === 'win32') return ['--use-compress-program=zstd -T0 --long=30'] return ['--use-compress-program', 'zstdmt --long=30'] case CompressionMethod.ZstdWithoutLong: + if (process.platform === 'win32') return ['--use-compress-program=zstd -T0'] return ['--use-compress-program', 'zstdmt'] default: return ['-z'] @@ -133,15 +139,18 @@ export async function listTar( archivePath: string, compressionMethod: CompressionMethod ): Promise { - // --d: Decompress. + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' // --long=#: Enables long distance matching with # bits. // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. // Using 30 here because we also support 32-bit self-hosted runners. function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: + if (process.platform === 'win32') return ['--use-compress-program=zstd -d --long=30'] return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: + if (process.platform === 'win32') return ['--use-compress-program=zstd -d'] return ['--use-compress-program', 'unzstd'] default: return ['-z'] From f8835dcb6785d3d89d39079cb683023e457eeddc Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Sat, 13 Aug 2022 18:46:24 +0000 Subject: [PATCH 2/8] Bump internal package @actions/cache version to fix #888 and #891 in actions/cache repo --- RELEASES.md | 5 ++++- package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/RELEASES.md b/RELEASES.md index 8fc7ba2..749afb3 100644 --- a/RELEASES.md +++ b/RELEASES.md @@ -80,4 +80,7 @@ - Added 1 hour timeout for the download stuck issue [#810](https://github.com/actions/cache/issues/810). ### 3.0.3 -- Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810). \ No newline at end of file +- Bug fixes for download stuck issue [#810](https://github.com/actions/cache/issues/810). + +### 3.0.4 +- Fix zstd not working for windows on gnu tar in issues [#888](https://github.com/actions/cache/issues/888) and [#891](https://github.com/actions/cache/issues/891). \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index 87d24d4..0ea84f4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@actions/cache", - "version": "3.0.3", + "version": "3.0.4", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "@actions/cache", - "version": "3.0.3", + "version": "3.0.4", "license": "MIT", "dependencies": { "@actions/core": "^1.2.6", diff --git a/package.json b/package.json index 476e856..6a3d476 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@actions/cache", - "version": "3.0.3", + "version": "3.0.4", "preview": true, "description": "Actions cache lib", "keywords": [ From cd6fe2862c14d7a239b9f89f6c273094e4b1ac03 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 17 Aug 2022 07:47:49 +0000 Subject: [PATCH 3/8] Fix linting --- src/internal/tar.ts | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/internal/tar.ts b/src/internal/tar.ts index 7c45a93..0733577 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -68,10 +68,14 @@ export async function extractTar( function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: - if (process.platform === 'win32') return ['--use-compress-program=zstd -d --long=30'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -d --long=30'] + } return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') return ['--use-compress-program=zstd -d'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -d'] + } return ['--use-compress-program', 'unzstd'] default: return ['-z'] @@ -110,10 +114,14 @@ export async function createTar( function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: - if (process.platform === 'win32') return ['--use-compress-program=zstd -T0 --long=30'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -T0 --long=30'] + } return ['--use-compress-program', 'zstdmt --long=30'] case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') return ['--use-compress-program=zstd -T0'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -T0'] + } return ['--use-compress-program', 'zstdmt'] default: return ['-z'] @@ -147,10 +155,14 @@ export async function listTar( function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: - if (process.platform === 'win32') return ['--use-compress-program=zstd -d --long=30'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -d --long=30'] + } return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') return ['--use-compress-program=zstd -d'] + if (process.platform === 'win32') { + return ['--use-compress-program=zstd -d'] + } return ['--use-compress-program', 'unzstd'] default: return ['-z'] From 1e4d71c2cac3a7151ff1417ba35ab1fe43020063 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Wed, 17 Aug 2022 08:21:09 +0000 Subject: [PATCH 4/8] Fix failing test cases on windows --- __tests__/tar.test.ts | 8 ++++---- src/internal/tar.ts | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/__tests__/tar.test.ts b/__tests__/tar.test.ts index b05c67e..e4233bc 100644 --- a/__tests__/tar.test.ts +++ b/__tests__/tar.test.ts @@ -50,7 +50,7 @@ test('zstd extract tar', async () => { `"${defaultTarPath}"`, [ '--use-compress-program', - 'unzstd --long=30', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-xf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P', @@ -140,7 +140,7 @@ test('zstd create tar', async () => { [ '--posix', '--use-compress-program', - 'zstdmt --long=30', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30', '-cf', IS_WINDOWS ? CacheFilename.Zstd.replace(/\\/g, '/') : CacheFilename.Zstd, '--exclude', @@ -210,7 +210,7 @@ test('zstd list tar', async () => { `"${defaultTarPath}"`, [ '--use-compress-program', - 'unzstd --long=30', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' @@ -235,7 +235,7 @@ test('zstdWithoutLong list tar', async () => { `"${defaultTarPath}"`, [ '--use-compress-program', - 'unzstd', + IS_WINDOWS ? 'zstd -d' : 'unzstd', '-tf', IS_WINDOWS ? archivePath.replace(/\\/g, '/') : archivePath, '-P' diff --git a/src/internal/tar.ts b/src/internal/tar.ts index 0733577..cb563a0 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -69,12 +69,12 @@ export async function extractTar( switch (compressionMethod) { case CompressionMethod.Zstd: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -d --long=30'] + return ['--use-compress-program', 'zstd -d --long=30'] } return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -d'] + return ['--use-compress-program', 'zstd -d'] } return ['--use-compress-program', 'unzstd'] default: @@ -115,12 +115,12 @@ export async function createTar( switch (compressionMethod) { case CompressionMethod.Zstd: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -T0 --long=30'] + return ['--use-compress-program', 'zstd -T0 --long=30'] } return ['--use-compress-program', 'zstdmt --long=30'] case CompressionMethod.ZstdWithoutLong: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -T0'] + return ['--use-compress-program', 'zstd -T0'] } return ['--use-compress-program', 'zstdmt'] default: @@ -156,12 +156,12 @@ export async function listTar( switch (compressionMethod) { case CompressionMethod.Zstd: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -d --long=30'] + return ['--use-compress-program', 'zstd -d --long=30'] } return ['--use-compress-program', 'unzstd --long=30'] case CompressionMethod.ZstdWithoutLong: if (process.platform === 'win32') { - return ['--use-compress-program=zstd -d'] + return ['--use-compress-program', 'zstd -d'] } return ['--use-compress-program', 'unzstd'] default: From 09244d83b7b154dd7debca1c801163de9c895283 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 18 Aug 2022 11:01:22 +0000 Subject: [PATCH 5/8] Common getCompressionMethod for listTar and extractTar --- src/internal/constants.ts | 6 ++ src/internal/tar.ts | 113 ++++++++++++++++---------------------- 2 files changed, 54 insertions(+), 65 deletions(-) diff --git a/src/internal/constants.ts b/src/internal/constants.ts index 2f78d32..8686a63 100644 --- a/src/internal/constants.ts +++ b/src/internal/constants.ts @@ -11,6 +11,12 @@ export enum CompressionMethod { Zstd = 'zstd' } +export enum TarOperation { + Create = 'create', + List = 'list', + Extract = 'extract' +} + // The default number of retry attempts. export const DefaultRetryAttempts = 2 diff --git a/src/internal/tar.ts b/src/internal/tar.ts index cb563a0..34229cc 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -3,7 +3,9 @@ import * as io from '@actions/io' import {existsSync, writeFileSync} from 'fs' import * as path from 'path' import * as utils from './cacheUtils' -import {CompressionMethod} from './constants' +import {CompressionMethod, TarOperation} from './constants' + +const IS_WINDOWS = process.platform === 'win32' async function getTarPath( args: string[], @@ -54,6 +56,41 @@ function getWorkingDirectory(): string { return process.env['GITHUB_WORKSPACE'] ?? process.cwd() } +// Common function for extractTar and listTar to get the compression method +function getCompressionProgram(compressionMethod: CompressionMethod): string[] { + // -d: Decompress. + // unzstd is equivalent to 'zstd -d' + // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. + // Using 30 here because we also support 32-bit self-hosted runners. + switch (compressionMethod) { + case CompressionMethod.Zstd: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + ] + case CompressionMethod.ZstdWithoutLong: + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -d' : 'unzstd', + ] + default: + return ['-z'] + } +} + +export async function listTar( + archivePath: string, + compressionMethod: CompressionMethod +): Promise { + const args = [ + getCompressionProgram(compressionMethod), + '-tf', + archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), + '-P' + ] + await execTar(args, compressionMethod) +} + export async function extractTar( archivePath: string, compressionMethod: CompressionMethod @@ -61,28 +98,8 @@ export async function extractTar( // Create directory to extract tar into const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) - // -d: Decompress. - // unzstd is equivalent to 'zstd -d' - // --long=#: Enables long distance matching with # bits. Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -d --long=30'] - } - return ['--use-compress-program', 'unzstd --long=30'] - case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -d'] - } - return ['--use-compress-program', 'unzstd'] - default: - return ['-z'] - } - } const args = [ - ...getCompressionProgram(), + getCompressionProgram(compressionMethod), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', @@ -114,15 +131,15 @@ export async function createTar( function getCompressionProgram(): string[] { switch (compressionMethod) { case CompressionMethod.Zstd: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -T0 --long=30'] - } - return ['--use-compress-program', 'zstdmt --long=30'] + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' + ] case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -T0'] - } - return ['--use-compress-program', 'zstdmt'] + return [ + '--use-compress-program', + IS_WINDOWS ? 'zstd -T0' : 'zstdmt' + ] default: return ['-z'] } @@ -141,38 +158,4 @@ export async function createTar( manifestFilename ] await execTar(args, compressionMethod, archiveFolder) -} - -export async function listTar( - archivePath: string, - compressionMethod: CompressionMethod -): Promise { - // -d: Decompress. - // unzstd is equivalent to 'zstd -d' - // --long=#: Enables long distance matching with # bits. - // Maximum is 30 (1GB) on 32-bit OS and 31 (2GB) on 64-bit. - // Using 30 here because we also support 32-bit self-hosted runners. - function getCompressionProgram(): string[] { - switch (compressionMethod) { - case CompressionMethod.Zstd: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -d --long=30'] - } - return ['--use-compress-program', 'unzstd --long=30'] - case CompressionMethod.ZstdWithoutLong: - if (process.platform === 'win32') { - return ['--use-compress-program', 'zstd -d'] - } - return ['--use-compress-program', 'unzstd'] - default: - return ['-z'] - } - } - const args = [ - ...getCompressionProgram(), - '-tf', - archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), - '-P' - ] - await execTar(args, compressionMethod) -} +} \ No newline at end of file From ec892fc29ae535aca910920fbc6d85778cc2cf8b Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 18 Aug 2022 11:10:41 +0000 Subject: [PATCH 6/8] Remove unused constants --- src/internal/constants.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/internal/constants.ts b/src/internal/constants.ts index 8686a63..2f78d32 100644 --- a/src/internal/constants.ts +++ b/src/internal/constants.ts @@ -11,12 +11,6 @@ export enum CompressionMethod { Zstd = 'zstd' } -export enum TarOperation { - Create = 'create', - List = 'list', - Extract = 'extract' -} - // The default number of retry attempts. export const DefaultRetryAttempts = 2 From 8765fae9d1bb5cf5eb4b4bfa038cf4ea435dd568 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 18 Aug 2022 11:22:13 +0000 Subject: [PATCH 7/8] Fix spread operator for getCompressionMethod --- src/internal/tar.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/internal/tar.ts b/src/internal/tar.ts index 3dee7fd..cf0ccd0 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -83,7 +83,7 @@ export async function listTar( compressionMethod: CompressionMethod ): Promise { const args = [ - getCompressionProgram(compressionMethod), + ...getCompressionProgram(compressionMethod), '-tf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P' @@ -99,7 +99,7 @@ export async function extractTar( const workingDirectory = getWorkingDirectory() await io.mkdirP(workingDirectory) const args = [ - getCompressionProgram(compressionMethod), + ...getCompressionProgram(compressionMethod), '-xf', archivePath.replace(new RegExp(`\\${path.sep}`, 'g'), '/'), '-P', From 0c62886e6af141340d77017248640b075fe7af69 Mon Sep 17 00:00:00 2001 From: Lovepreet Singh Date: Thu, 18 Aug 2022 11:44:15 +0000 Subject: [PATCH 8/8] Fix linting issues --- src/internal/tar.ts | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/internal/tar.ts b/src/internal/tar.ts index cf0ccd0..2e28ca1 100644 --- a/src/internal/tar.ts +++ b/src/internal/tar.ts @@ -66,13 +66,10 @@ function getCompressionProgram(compressionMethod: CompressionMethod): string[] { case CompressionMethod.Zstd: return [ '--use-compress-program', - IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30', + IS_WINDOWS ? 'zstd -d --long=30' : 'unzstd --long=30' ] case CompressionMethod.ZstdWithoutLong: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -d' : 'unzstd', - ] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -d' : 'unzstd'] default: return ['-z'] } @@ -136,10 +133,7 @@ export async function createTar( IS_WINDOWS ? 'zstd -T0 --long=30' : 'zstdmt --long=30' ] case CompressionMethod.ZstdWithoutLong: - return [ - '--use-compress-program', - IS_WINDOWS ? 'zstd -T0' : 'zstdmt' - ] + return ['--use-compress-program', IS_WINDOWS ? 'zstd -T0' : 'zstdmt'] default: return ['-z'] } @@ -158,4 +152,4 @@ export async function createTar( manifestFilename ] await execTar(args, compressionMethod, archiveFolder) -} \ No newline at end of file +}