const cp = require("child_process"); const fs = require("fs"); const GeneralUtils = require("./general"); const path = require("path"); const DEFAULT_TIMEOUT = process.env.FILESYSTEM_DEFAULT_TIMEOUT || 30000; /** * https://github.com/kubernetes/kubernetes/tree/master/pkg/util/mount * https://github.com/kubernetes/kubernetes/blob/master/pkg/util/mount/mount_linux.go */ class Filesystem { constructor(options = {}) { const filesystem = this; filesystem.options = options; options.paths = options.paths || {}; if (!options.paths.sudo) { options.paths.sudo = "/usr/bin/sudo"; } if (!options.executor) { options.executor = { spawn: cp.spawn, }; } } covertUnixSeparatorToWindowsSeparator(p) { return p.replaceAll(path.posix.sep, path.win32.sep); } /** * Attempt to discover if device is a block device * * @param {*} device */ async isBlockDevice(device) { const filesystem = this; // nfs paths if (!device.startsWith("/")) { return false; } // smb paths if (device.startsWith("//")) { return false; } const device_path = await filesystem.realpath(device); const blockdevices = await filesystem.getAllBlockDevices(); return blockdevices.some(async (i) => { if ((await filesystem.realpath(i.path)) == device_path) { return true; } return false; }); } /** * Attempt to discover if the device is a device-mapper device * * @param {*} device */ async isDeviceMapperDevice(device) { const filesystem = this; const isBlock = await filesystem.isBlockDevice(device); if (!isBlock) { return false; } device = await filesystem.realpath(device); return device.includes("dm-"); } async isDeviceMapperSlaveDevice(device) { const filesystem = this; device = await filesystem.realpath(device); } /** * Get all device-mapper devices (ie: dm-0, dm-1, dm-N...) */ async getAllDeviceMapperDevices() { const filesystem = this; let result; let devices = []; let args = [ "-c", 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', ]; try { result = await filesystem.exec("sh", args); for (const dm of result.stdout.trim().split("\n")) { if (dm.length < 1) { continue; } devices.push("/dev/" + dm.split(":")[0].trim()); } return devices; } catch (err) { throw err; } } async getAllDeviceMapperSlaveDevices() { const filesystem = this; let result; let args = [ "-c", 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', ]; let slaves = []; try { result = await filesystem.exec("sh", args); for (const dm of result.stdout.trim().split("\n")) { if (dm.length < 1) { continue; } const realDevices = dm .split(":")[1] .split(" ") .map((value) => { return "/dev/" + value.trim(); }); slaves.push(...realDevices); } return slaves; } catch (err) { throw err; } } /** * Get all slave devices connected to a device-mapper device * * @param {*} device */ async getDeviceMapperDeviceSlaves(device) { const filesystem = this; device = await filesystem.realpath(device); let device_info = await filesystem.getBlockDevice(device); const slaves = []; let result; let args = [`/sys/block/${device_info.kname}/slaves/`]; try { result = await filesystem.exec("ls", args); for (const entry of result.stdout.split("\n")) { if (entry.trim().length < 1) { continue; } slaves.push("/dev/" + entry.trim()); } return slaves; } catch (err) { throw err; } } async getDeviceMapperDeviceFromSlaves(slaves, matchAll = true) { const filesystem = this; let result; // get mapping of dm devices to real devices let args = [ "-c", 'for file in $(ls -la /dev/mapper/* | grep "\\->" | grep -oP "\\-> .+" | grep -oP " .+"); do echo $(F=$(echo $file | grep -oP "[a-z0-9-]+");echo $F":"$(ls "/sys/block/${F}/slaves/");); done;', ]; result = await filesystem.exec("sh", args); for (const dm of result.stdout.trim().split("\n")) { if (dm.length < 1) { continue; } const dmDevice = "/dev/" + dm.split(":")[0].trim(); const realDevices = dm .split(":")[1] .split(" ") .map((value) => { return "/dev/" + value.trim(); }); const intersectDevices = slaves.filter((value) => realDevices.includes(value) ); if (matchAll === false && intersectDevices.length > 0) { return dmDevice; } // if all 3 have the same elements we have a winner if ( intersectDevices.length == realDevices.length && realDevices.length == slaves.length ) { return dmDevice; } } } /** * create symlink * * @param {*} device */ async symlink(target, link, options = []) { const filesystem = this; let args = ["-s"]; args = args.concat(options); args = args.concat([target, link]); try { await filesystem.exec("ln", args); } catch (err) { throw err; } } async isSymbolicLink(path) { return fs.lstatSync(path).isSymbolicLink(); } /** * remove file * * @param {*} device */ async rm(options = []) { const filesystem = this; let args = []; args = args.concat(options); try { await filesystem.exec("rm", args); } catch (err) { throw err; } } /** * touch a path * @param {*} path */ async touch(path, options = []) { const filesystem = this; let args = []; args = args.concat(options); args.push(path); try { await filesystem.exec("touch", args); } catch (err) { throw err; } } /** * touch a path * @param {*} path */ async dirname(path) { const filesystem = this; let args = []; args.push(path); let result; try { result = await filesystem.exec("dirname", args); return result.stdout.trim(); } catch (err) { throw err; } } /** * lsblk -a -b -l -J -O */ async getAllBlockDevices() { const filesystem = this; let args = ["-a", "-b", "-l", "-J", "-O"]; let result; try { result = await filesystem.exec("lsblk", args); const parsed = JSON.parse(result.stdout); return parsed.blockdevices; } catch (err) { throw err; } } /** * lsblk -a -b -l -J -O */ async getBlockDevice(device) { const filesystem = this; device = await filesystem.realpath(device); let args = ["-a", "-b", "-J", "-O"]; args.push(device); let result; try { result = await filesystem.exec("lsblk", args); const parsed = JSON.parse(result.stdout); if (parsed.blockdevices.length != 1) { throw new Error(`cannot properly find device: ${device}`); } return parsed.blockdevices[0]; } catch (err) { throw err; } } /** * * @param {*} device * @returns */ async getBlockDeviceLargestPartition(device) { const filesystem = this; let block_device_info = await filesystem.getBlockDevice(device); if (block_device_info.children) { let child; for (const child_i of block_device_info.children) { if (child_i.type == "part") { if (!child) { child = child_i; } else { if (child_i.size > child.size) { child = child_i; } } } } return `${child.path}`; } } /** * * @param {*} device * @returns */ async getBlockDeviceLastPartition(device) { const filesystem = this; let block_device_info = await filesystem.getBlockDevice(device); if (block_device_info.children) { let child; for (const child_i of block_device_info.children) { if (child_i.type == "part") { if (!child) { child = child_i; } else { let minor = child["maj:min"].split(":")[1]; let minor_i = child_i["maj:min"].split(":")[1]; if (minor_i > minor) { child = child_i; } } } } return `${child.path}`; } } /** * * @param {*} device * @returns */ async getBlockDevicePartitionCount(device) { const filesystem = this; let count = 0; let block_device_info = await filesystem.getBlockDevice(device); if (block_device_info.children) { for (const child_i of block_device_info.children) { if (child_i.type == "part") { count++; } } } return count; } async getBlockDeviceHasParitionTable(device) { const filesystem = this; let block_device_info = await filesystem.getBlockDevice(device); return block_device_info.pttype ? true : false; } /** * DOS * - type=83 = Linux * - type=07 = HPFS/NTFS/exFAT * * GPT * - type=0FC63DAF-8483-4772-8E79-3D69D8477DE4 = linux * - type=EBD0A0A2-B9E5-4433-87C0-68B6B72699C7 = ntfs * - type=C12A7328-F81F-11D2-BA4B-00A0C93EC93B = EFI * * @param {*} device * @param {*} label * @param {*} type */ async partitionDevice( device, label = "gpt", type = "0FC63DAF-8483-4772-8E79-3D69D8477DE4" ) { const filesystem = this; let args = [device]; let result; try { result = await filesystem.exec("sfdisk", args, { stdin: `label: ${label}\n`, }); result = await filesystem.exec("sfdisk", args, { stdin: `type=${type}\n`, }); } catch (err) { throw err; } } /** * mimic the behavior of partitioning a new data drive in windows directly * * https://en.wikipedia.org/wiki/Microsoft_Reserved_Partition * * @param {*} device */ async partitionDeviceWindows(device) { const filesystem = this; let args = [device]; let result; let block_device_info = await filesystem.getBlockDevice(device); //let sixteen_megabytes = 16777216; //let thirtytwo_megabytes = 33554432; //let onehundredtwentyeight_megabytes = 134217728; let msr_partition_size = "16M"; let label = "gpt"; let msr_guid = "E3C9E316-0B5C-4DB8-817D-F92DF00215AE"; let ntfs_guid = "EBD0A0A2-B9E5-4433-87C0-68B6B72699C7"; if (block_device_info.type != "disk") { throw new Error( `cannot partition device of type: ${block_device_info.type}` ); } /** * On drives less than 16GB in size, the MSR is 32MB. * On drives greater than or equal two 16GB, the MSR is 128 MB. * It is only 128 MB for Win 7/8 ( On drives less than 16GB in size, the MSR is 32MB ) & 16 MB for win 10! */ let msr_partition_size_break = 17179869184; // 16GB // TODO: this size may be sectors so not really disk size in terms of GB if (block_device_info.size >= msr_partition_size_break) { // ignoring for now, appears windows 10+ use 16MB always //msr_partition_size = "128M"; } try { result = await filesystem.exec("sfdisk", args, { stdin: `label: ${label}\n`, }); // must send ALL partitions at once (newline separated), cannot send them 1 at a time result = await filesystem.exec("sfdisk", args, { stdin: `size=${msr_partition_size},type=${msr_guid}\ntype=${ntfs_guid}\n`, }); } catch (err) { throw err; } } /** * * @param {*} device */ async deviceIsFormatted(device) { const filesystem = this; let result; try { result = await filesystem.getBlockDevice(device); return result.fstype ? true : false; } catch (err) { throw err; } } async deviceIsIscsi(device) { const filesystem = this; let result; do { if (result) { device = `/dev/${result.pkname}`; } result = await filesystem.getBlockDevice(device); } while (result.pkname); return result && result.tran == "iscsi"; } async getBlockDeviceParent(device) { const filesystem = this; let result; do { if (result) { device = `/dev/${result.pkname}`; } result = await filesystem.getBlockDevice(device); } while (result.pkname); return result; } /** * blkid -p -o export * * @param {*} device */ async getDeviceFilesystemInfo(device) { const filesystem = this; let args = ["-p", "-o", "export", device]; let result; try { result = await filesystem.exec("blkid", args); const entries = result.stdout.trim().split("\n"); const properties = {}; let fields, key, value; entries.forEach((entry) => { fields = entry.split("="); key = fields[0].toLowerCase(); value = fields[1]; properties[key] = value; }); return properties; } catch (err) { throw err; } } /** * mkfs. [] device * * @param {*} device * @param {*} fstype * @param {*} options */ async formatDevice(device, fstype, options = []) { const filesystem = this; let args = []; args = args.concat(options); switch (fstype) { case "vfat": args = args.concat(["-I"]); break; } args.push(device); let result; try { result = await filesystem.exec("mkfs." + fstype, args); return result; } catch (err) { throw err; } } async realpath(path) { const filesystem = this; let args = [path]; let result; try { result = await filesystem.exec("realpath", args); return result.stdout.trim(); } catch (err) { throw err; } } async rescanDevice(device) { const filesystem = this; let result; let device_name; result = await filesystem.isBlockDevice(device); if (!result) { throw new Error( `cannot rescan device ${device} because it is not a block device` ); } let is_device_mapper_device = await filesystem.isDeviceMapperDevice(device); result = await filesystem.realpath(device); if (is_device_mapper_device) { // multipath -r /dev/dm-0 result = await filesystem.exec("multipath", ["-r", device]); } else { device_name = result.split("/").pop(); // echo 1 > /sys/block/sdb/device/rescan const sys_file = `/sys/block/${device_name}/device/rescan`; // node-local devices cannot be rescanned, so ignore if (await filesystem.pathExists(sys_file)) { console.log(`executing filesystem command: echo 1 > ${sys_file}`); fs.writeFileSync(sys_file, "1"); } } } async expandPartition(device) { const filesystem = this; const command = "growpart"; const args = []; let block_device_info = await filesystem.getBlockDevice(device); let device_fs_info = await filesystem.getDeviceFilesystemInfo(device); let growpart_partition = device_fs_info["part_entry_number"]; let parent_block_device = await filesystem.getBlockDeviceParent(device); args.push(parent_block_device.path, growpart_partition); try { await filesystem.exec(command, args); } catch (err) { if ( err.code == 1 && err.stdout && err.stdout.includes("could only be grown by") ) { return; } } } /** * expand a given filesystem * * @param {*} device * @param {*} fstype * @param {*} options */ async expandFilesystem(device, fstype, options = []) { const filesystem = this; let command; let args = []; let result; switch (fstype.toLowerCase()) { case "btrfs": command = "btrfs"; //args = args.concat(options); args = args.concat(["filesystem", "resize", "max"]); args.push(device); // in this case should be a mounted path break; case "exfat": // https://github.com/exfatprogs/exfatprogs/issues/134 return; case "ext4": case "ext3": case "ext4dev": command = "resize2fs"; args = args.concat(options); args.push(device); break; case "ntfs": // must be unmounted command = "ntfsresize"; await filesystem.exec(command, ["-c", device]); await filesystem.exec(command, ["-n", device]); args = args.concat("-P", "-f"); args = args.concat(options); //args = args.concat(["-s", "max"]); args.push(device); break; case "xfs": command = "xfs_growfs"; args = args.concat(options); args.push(device); // in this case should be a mounted path break; case "vfat": // must be unmounted command = "fatresize"; args = args.concat(options); args = args.concat(["-s", "max"]); args.push(device); break; } try { result = await filesystem.exec(command, args); // must clear the dirty bit after resize if (fstype.toLowerCase() == "ntfs") { await filesystem.exec("ntfsfix", ["-d", device]); } return result; } catch (err) { throw err; } } /** * check a given filesystem * * fsck [options] -- [fs-options] [ ...] * * @param {*} device * @param {*} fstype * @param {*} options * @param {*} fsoptions */ async checkFilesystem(device, fstype, options = [], fsoptions = []) { const filesystem = this; let command; let args = []; let result; switch (fstype.toLowerCase()) { case "btrfs": command = "btrfs"; args = args.concat(options); args.push("check"); args.push(device); break; case "ext4": case "ext3": case "ext4dev": command = "fsck"; args = args.concat(options); args.push(device); args.push("--"); args = args.concat(fsoptions); args.push("-f"); args.push("-p"); break; case "ntfs": /** * -b, --clear-bad-sectors Clear the bad sector list * -d, --clear-dirty Clear the volume dirty flag */ command = "ntfsfix"; args.puuh("-d"); args.push(device); break; case "xfs": command = "xfs_repair"; args = args.concat(["-o", "force_geometry"]); args = args.concat(options); args.push(device); break; default: command = "fsck"; args = args.concat(options); args.push(device); args.push("--"); args = args.concat(fsoptions); break; } try { result = await filesystem.exec(command, args); return result; } catch (err) { throw err; } } /** * mkdir [] * * @param {*} path * @param {*} options */ async mkdir(path, options = []) { const filesystem = this; let args = []; args = args.concat(options); args.push(path); try { await filesystem.exec("mkdir", args); } catch (err) { throw err; } return true; } /** * rmdir [] * * @param {*} path * @param {*} options */ async rmdir(path, options = []) { const filesystem = this; let args = []; args = args.concat(options); args.push(path); try { await filesystem.exec("rmdir", args); } catch (err) { throw err; } return true; } async getInodeInfo(path) { const filesystem = this; let args = ["-i"]; let result; args.push(path); try { result = await filesystem.exec("df", args); if (result.code == 0) { result = result.stdout.split("\n")[1].replace(/\s\s+/g, " "); let parts = result.split(" "); return { device: parts[0], mount_path: parts[5], inodes_total: parseInt(parts[1]), inodes_used: parseInt(parts[2]), inodes_free: parseInt(parts[3]), }; } } catch (err) { throw err; } } /** * * @param {*} path */ async pathExists(path) { let result = false; try { await GeneralUtils.retry( 10, 200, () => { fs.statSync(path); }, { retryCondition: (err) => { if (err.code == "UNKNOWN") { return true; } return false; }, } ); result = true; } catch (err) { if (err.code !== "ENOENT") { throw err; } } return result; } exec(command, args, options = {}) { if (!options.hasOwnProperty("timeout")) { // TODO: cannot use this as fsck etc are too risky to kill //options.timeout = DEFAULT_TIMEOUT; } let stdin; if (options.stdin) { stdin = options.stdin; delete options.stdin; } const filesystem = this; args = args || []; if (filesystem.options.sudo) { args.unshift(command); command = filesystem.options.paths.sudo; } let command_log = `${command} ${args.join(" ")}`.trim(); if (stdin) { command_log = `echo '${stdin}' | ${command_log}` .trim() .replace(/\n/, "\\n"); } console.log("executing filesystem command: %s", command_log); return new Promise((resolve, reject) => { const child = filesystem.options.executor.spawn(command, args, options); let stdout = ""; let stderr = ""; child.on("spawn", function () { if (stdin) { child.stdin.setEncoding("utf-8"); child.stdin.write(stdin); child.stdin.end(); } }); child.stdout.on("data", function (data) { stdout = stdout + data; }); child.stderr.on("data", function (data) { stderr = stderr + data; }); child.on("close", function (code) { const result = { code, stdout, stderr, timeout: false }; // timeout scenario if (code === null) { result.timeout = true; reject(result); } if (code) { console.log( "failed to execute filesystem command: %s, response: %j", [command].concat(args).join(" "), result ); reject(result); } else { resolve(result); } }); }); } } module.exports.Filesystem = Filesystem;