Optimizing enumeration

Files/Program.cs

@@ -13,7 +13,7 @@ using System.Linq;
 using System.Reflection;
 using System.Reflection.Emit;
 using Dapper;
-using System.Security.Cryptography;
+using Mono.Unix.Native;
 
 namespace Files
 {
@@ -113,9 +113,11 @@ namespace Files
                         if (ct.IsCancellationRequested)
                             return;
 
-                        var sameSize = connection.Query<DbRecord>("SELECT name, size, inode FROM files WHERE size = @size",
+                        var sameSize = connection.Query<UnixFileRecord>("SELECT name, size, inode FROM files WHERE size = @size",
                             new { potentialFile.size }).ToList();
 
+                        sameSize.CalculateHashes();
+
                         var recordsWithErrors = sameSize
                             .Where(r => !r.Hash.HasValue);
 
@@ -137,9 +139,9 @@ namespace Files
 
                             var records = grp.OrderByDescending(r => r.FileInfo.LinkCount).ToList();
 
-                            DbRecord head = records.First();
+                            UnixFileRecord head = records.First();
-                            var tail = records.Skip(1).Where(r => r.Inode != head.Inode).ToList();
+                            var tail = records.Skip(1).Where(r => r.INode != head.INode).ToList();
-                            var tailWithDuplicates = records.Skip(1).Where(r => r.Inode == head.Inode).ToList();
+                            var tailWithDuplicates = records.Skip(1).Where(r => r.INode == head.INode).ToList();
 
                             ByteSize totalSize = records.Distinct(new DbRecordEqualityComparerByINode()).Sum(a => a.Size) - head.Size;
 
@@ -207,9 +209,13 @@ namespace Files
         private static async Task InitializeDb(SqliteConnection connection)
         {
             await connection.ExecuteAsync(
-                "CREATE TABLE IF NOT EXISTS files " +
+                "CREATE TABLE IF NOT EXISTS files (" +
-                "(name TEXT PRIMARY KEY, size INTEGER NOT NULL, inode INTEGER NOT NULL);");
+                        "name TEXT PRIMARY KEY, " +
+                        "size INTEGER NOT NULL, " +
+                        "inode INTEGER NOT NULL, " +
+                        "hash TEXT);");
             await connection.ExecuteAsync("CREATE INDEX IF NOT EXISTS idx_files_size ON files(size);");
+            await connection.ExecuteAsync("CREATE INDEX IF NOT EXISTS idx_files_inode ON files(inode);");
         }
 
         private static async Task Main(string[] args)
@@ -218,6 +224,7 @@ namespace Files
             var hardlinkOption = new Option<bool>(new[] { "--hardlink", "-l" }, "Hardlink duplicates");
             var databaseOption = new Option<bool>(new[] { "--keep", "-k" }, () => true, "Keep database.");
             var scanOption = new Option<bool>(new[] { "--no-scan" }, "Do not scan file system. Reuse database.");
+            var dbFileOption = new Option<FileInfo>(new[] { "--database", "-db" }, "Store database in file.");
             var directoryArgument = new Argument<DirectoryInfo>(
                 result => new DirectoryInfo("./"), isDefault: true)
             {
@@ -232,6 +239,7 @@ namespace Files
                 hardlinkOption,
                 databaseOption,
                 scanOption,
+                dbFileOption,
                 directoryArgument,
             };
 
@@ -249,16 +257,268 @@ namespace Files
                 InitialDirectory = dir,
                 KeepDatabase = result.ValueForOption(databaseOption),
                 SkipFileScanning = result.ValueForOption(scanOption),
+                DatabaseFile = result.ValueForOption(dbFileOption),
             };
 
             rootCommand.Handler = CommandHandler.Create<CancellationToken>(
                 async ct =>
                 {
-                    await IndexFiles(config, ct);
+                    //await IndexFiles(config, ct);
+                    await Begin(config, ct);
                 });
 
             await rootCommand.InvokeAsync(args);
         }
+
+        private static async Task Begin(Configuration configuration, CancellationToken ct) =>
+            await AnsiConsole.Status()
+                .StartAsync("Initializing...", async ctx =>
+                {
+                    string dbFileName = configuration.DatabaseFile?.FullName ?? ":memory:";
+                    await using var connection = new SqliteConnection($"Data Source={dbFileName};");
+                    connection.Open();
+                    await using var transaction = await connection.BeginTransactionAsync(ct);
+
+                    try
+                    {
+                        await InitializeDb(connection);
+
+                        if (!configuration.SkipFileScanning)
+                        {
+                            await ScanFiles(configuration, connection, ctx, ct);
+                        }
+
+                        FindDuplicates(configuration, connection, ctx, ct);
+
+                        await transaction.CommitAsync(ct);
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        await transaction.RollbackAsync();
+                        AnsiConsole.WriteLine("Canceled!");
+                    }
+                    catch (Exception exception)
+                    {
+                        await transaction.RollbackAsync();
+                        AnsiConsole.WriteException(exception);
+                    }
+                });
+
+        private static async Task ScanFiles(Configuration configuration, SqliteConnection sqliteConnection,
+            StatusContext statusContext, CancellationToken ct)
+        {
+            UnixFileSystemEnumerator.FilterEnumeratorDelegate filter = (directory, entry, entryType, errno) => true;
+
+            var pathEnumerable = UnixFileSystemEnumerator.EnumeratePaths(
+                configuration.InitialDirectory.ToString(),
+                filter,
+                ct);
+
+            foreach ((string entryPath, byte entryType, Errno errno) in pathEnumerable)
+            {
+                if (errno != 0)
+                {
+                    string errorDescription = UnixMarshal.GetErrorDescription(errno);
+                    string safeErrorDescription = errorDescription
+                        .Replace("[", "[[")
+                        .Replace("]", "]]");
+                    string safePath = entryPath
+                        .Replace("[", "[[")
+                        .Replace("]", "]]");
+                    AnsiConsole.MarkupLine($"[red]:cross_mark: {safeErrorDescription}:[/] :file_folder: {safePath}");
+
+                    continue;
+                }
+
+                string entryTypeEmoji = entryType switch
+                {
+                    DirentType.DT_DIR => Emoji.Known.FileFolder,
+                    DirentType.DT_REG => Emoji.Known.PageFacingUp,
+                    DirentType.DT_LNK => Emoji.Known.Link,
+                    DirentType.DT_BLK => Emoji.Known.ComputerDisk,
+                    DirentType.DT_CHR => Emoji.Known.Keyboard,
+                    DirentType.DT_FIFO => Emoji.Known.PButton,
+                    DirentType.DT_SOCK => Emoji.Known.ElectricPlug,
+                    DirentType.DT_UNKNOWN => Emoji.Known.Potato,
+                    _ => Emoji.Known.PileOfPoo,
+                };
+
+                if (!UnixFileSystemEnumerator.IsOfTarget(entryType, SearchTarget.DirectoriesAndFiles))
+                {
+                    if(!configuration.BeVerbose) continue;
+
+                    string safePath = entryPath
+                        .Replace("[", "[[")
+                        .Replace("]", "]]");
+
+                    string fileType = entryType switch
+                    {
+                        DirentType.DT_DIR => "Directory",
+                        DirentType.DT_REG => "Regular file",
+                        DirentType.DT_LNK => "Symbolic link",
+                        DirentType.DT_BLK => "Block device",
+                        DirentType.DT_CHR => "Character device",
+                        DirentType.DT_FIFO => "Named pipe",
+                        DirentType.DT_SOCK => "Socket",
+                        DirentType.DT_UNKNOWN => "UNKNOWN",
+                        _ => "WRONG",
+                    };
+
+                    AnsiConsole.MarkupLine($"[yellow]{Emoji.Known.FastForwardButton} {fileType}:[/] {entryTypeEmoji} {safePath}");
+
+                    continue;
+                }
+
+                UnixFileSystemInfo entry = entryType switch
+                {
+                    DirentType.DT_REG => new UnixFileInfo(entryPath),
+                    DirentType.DT_DIR => new UnixDirectoryInfo(entryPath),
+                    DirentType.DT_LNK => new UnixSymbolicLinkInfo(entryPath),
+                    _ => throw new FileLoadException($"Cannot scan {entryTypeEmoji} {entryPath}"),
+                };
+
+                if (!entry.GetValid())
+                {
+                    string errorDescription = UnixMarshal.GetErrorDescription(Stdlib.GetLastError());
+                    string safePath = entryPath
+                        .Replace("[", "[[")
+                        .Replace("]", "]]");
+                    AnsiConsole.MarkupLine($"[red]:cross_mark: {errorDescription}:[/] {entryTypeEmoji} {safePath}");
+                    continue;
+                }
+
+                string safeEntryPath = entryPath
+                    .Replace("[", "[[")
+                    .Replace("]", "]]");
+
+                if (entry.GetType() == typeof(UnixFileInfo)) // Faster than "is"
+                {
+                    var file = (UnixFileInfo) entry;
+                    var record = new UnixFileRecord(file);
+
+                    await sqliteConnection.ExecuteAsync("INSERT OR REPLACE INTO files (name, size, inode) VALUES (@Name, @Size, @INode);", record);
+
+                    if (configuration.BeVerbose)
+                        AnsiConsole.MarkupLine($"[green]:check_mark: OK:[/] {entryTypeEmoji} {safeEntryPath}");
+                }
+                else if (entry.GetType() == typeof(UnixDirectoryInfo)) // Faster than "is"
+                {
+                    var directory = (UnixDirectoryInfo)entry;
+                    statusContext.Status(safeEntryPath);
+                }
+                else if (entry.GetType() == typeof(UnixSymbolicLinkInfo)) // Faster than "is"
+                {
+                    var symLink = (UnixSymbolicLinkInfo)entry;
+                }
+
+                if (ct.IsCancellationRequested)
+                    return;
+            }
+        }
+
+        private static void FindDuplicates(Configuration configuration,
+            SqliteConnection connection, StatusContext ctx,
+            CancellationToken ct)
+        {
+            ctx.Status("Finding duplicates...");
+            ctx.Spinner(Spinner.Known.Aesthetic);
+
+            var potential = connection.Query<(int cnt, long size)>(
+                "SELECT COUNT(*) cnt, size FROM files WHERE size != 0 GROUP BY size HAVING cnt > 1 ORDER BY size * cnt DESC;");
+
+            foreach (var potentialFile in potential)
+            {
+                ct.ThrowIfCancellationRequested();
+
+                var sameSize = connection.Query<UnixFileRecord>("SELECT name, size, inode FROM files WHERE size = @size",
+                    new { potentialFile.size }).ToList();
+
+                var recordsWithErrors = sameSize
+                    .Where(r => !r.Hash.HasValue);
+
+                foreach (var dbRecord in recordsWithErrors)
+                {
+                    AnsiConsole.MarkupLine(
+                        $"[red]:cross_mark: NO_ACCESS:[/] :page_facing_up: {dbRecord.Name.Replace("[", "[[").Replace("]", "]]")}");
+                }
+
+                var equalGrouped = sameSize
+                    .Where(r => r.Hash.HasValue)
+                    .GroupBy(r => r.Hash)
+                    .Where(g => g.Count() > 1)
+                    .ToList();
+
+                foreach (var grp in equalGrouped)
+                {
+                    ct.ThrowIfCancellationRequested();
+
+                    var records = grp.OrderByDescending(r => r.FileInfo.LinkCount).ToList();
+
+                    UnixFileRecord head = records.First();
+                    var tail = records.Skip(1).Where(r => r.INode != head.INode).ToList();
+                    var tailWithDuplicates = records.Skip(1).Where(r => r.INode == head.INode).ToList();
+
+                    ByteSize totalSize = records.Distinct(new DbRecordEqualityComparerByINode()).Sum(a => a.Size) - head.Size;
+
+                    var root = new Tree((head.Size + totalSize).ToStringWithDecimalPrefixedShortUnitName() + " total.");
+                    root.AddNode(((ByteSize)head.Size).ToStringWithDecimalPrefixedShortUnitName() + " " +
+                                 head.Name.Replace("[", "[[").Replace("]", "]]"));
+                    foreach (var item in tail)
+                    {
+                        if (configuration.EnableLinking)
+                        {
+                            try
+                            {
+                                // First rename
+                                string tempFileName = item.FileInfo.FullName + ".to_hardlink";
+                                File.Move(item.FileInfo.FullName, tempFileName);
+
+                                try
+                                {
+                                    // Then hardlink
+                                    head.FileInfo.CreateLink(item.FileInfo.FullName);
+
+                                    // Then delete
+                                    File.Delete(tempFileName);
+
+                                    root.AddNode("[green]:check_mark:[/] " +
+                                                 item.Name.Replace("[", "[[").Replace("]", "]]"));
+                                }
+                                catch (Exception)
+                                {
+                                    File.Move(tempFileName, item.FileInfo.FullName);
+                                    throw;
+                                }
+                            }
+                            catch (Exception exception)
+                            {
+                                AnsiConsole.WriteException(exception, ExceptionFormats.ShortenEverything);
+                                root.AddNode("[red]:cross_mark:[/] " +
+                                             item.Name.Replace("[", "[[").Replace("]", "]]"));
+                            }
+                        }
+                        else
+                        {
+                            root.AddNode(((ByteSize)item.Size).ToStringWithDecimalPrefixedShortUnitName() + " " +
+                                         item.Name.Replace("[", "[[").Replace("]", "]]"));
+                        }
+                    }
+
+                    if (configuration.BeVerbose)
+                        foreach (var duplicate in tailWithDuplicates)
+                        {
+                            root.AddNode("[white]:anchor:[/] 0B " +
+                                         duplicate.Name.Replace("[", "[[").Replace("]", "]]"));
+                        }
+
+                    if (tail.Any() || configuration.BeVerbose)
+                    {
+                        AnsiConsole.Render(root);
+                        AnsiConsole.WriteLine();
+                    }
+                }
+            }
+        }
     }
 
     class Configuration
@@ -268,85 +528,54 @@ namespace Files
         public DirectoryInfo InitialDirectory { get; set; }
         public bool KeepDatabase { get; set; }
         public bool SkipFileScanning { get; set; }
+        public FileInfo DatabaseFile { get; set; }
     }
 
-    public class DbRecord
+    public class DbRecordEqualityComparerByINode : EqualityComparer<UnixFileRecord>
     {
-        private readonly Lazy<Guid?> _guid;
+        public override bool Equals(UnixFileRecord x, UnixFileRecord y)
-        private readonly Lazy<UnixFileInfo> _fileInfo;
-
-        public DbRecord()
         {
-            _guid = new Lazy<Guid?>(GetHash);
+            return x?.INode == y?.INode;
-            _fileInfo = new Lazy<UnixFileInfo>(GetFileInfo);
         }
 
-        public DbRecord(UnixFileInfo fileInfo)
+        public override int GetHashCode(UnixFileRecord obj)
         {
-            _guid = new Lazy<Guid?>(GetHash);
+            return obj.INode.GetHashCode();
-            _fileInfo = new Lazy<UnixFileInfo>(fileInfo);
-            Name = fileInfo.GetOriginalPath();
-            Size = fileInfo.Length;
-            Inode = fileInfo.Inode;
-        }
-
-        public string Name { get; set; }
-        public long Size { get; set; }
-        public long Inode { get; set; }
-
-        public Guid? Hash => _guid.Value;
-        public UnixFileInfo FileInfo => _fileInfo.Value;
-
-        private UnixFileInfo GetFileInfo() => new(Name);
-
-        private Guid? GetHash()
-        {
-            try
-            {
-                using FileStream stream = File.OpenRead(Name);
-                var md5 = MD5.Create();
-                var bytes = md5.ComputeHash(stream);
-                return new Guid(bytes);
-            }
-            catch
-            {
-                return null;
-            }
-        }
-    }
-
-    public class DbRecordEqualityComparerByINode : EqualityComparer<DbRecord>
-    {
-        public override bool Equals(DbRecord x, DbRecord y)
-        {
-            return x?.Inode == y?.Inode;
-        }
-
-        public override int GetHashCode(DbRecord obj)
-        {
-            return obj.Inode.GetHashCode();
         }
     }
 
     static class OriginalPathUnixFileSystemInfo
     {
         private static readonly Func<UnixFileSystemInfo, string> GetOriginalPathFunc;
+        private static readonly Func<UnixFileSystemInfo, bool> GetValidFunc;
 
         static OriginalPathUnixFileSystemInfo()
         {
             var method = new DynamicMethod("cheat", typeof(string), new[] { typeof(UnixFileSystemInfo) }, typeof(UnixFileSystemInfo), true);
             var il = method.GetILGenerator();
             il.Emit(OpCodes.Ldarg_0);
-            il.Emit(OpCodes.Castclass, typeof(UnixFileSystemInfo));
+            //il.Emit(OpCodes.Castclass, typeof(UnixFileSystemInfo));
             il.Emit(OpCodes.Callvirt, typeof(UnixFileSystemInfo)
                 .GetProperty("OriginalPath", BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic)
                 .GetGetMethod(true));
             il.Emit(OpCodes.Ret);
             GetOriginalPathFunc = (Func<UnixFileSystemInfo, string>)method.CreateDelegate(typeof(Func<UnixFileSystemInfo, string>));
+
+
+            var method2 = new DynamicMethod("cheat2", typeof(bool), new[] { typeof(UnixFileSystemInfo) }, typeof(UnixFileSystemInfo), true);
+            var il2 = method2.GetILGenerator();
+            il2.Emit(OpCodes.Ldarg_0);
+            //il2.Emit(OpCodes.Castclass, typeof(UnixFileSystemInfo));
+            il2.Emit(OpCodes.Ldfld, typeof(UnixFileSystemInfo)
+                .GetField("valid", BindingFlags.Instance | BindingFlags.NonPublic));
+            il2.Emit(OpCodes.Ret);
+            GetValidFunc = (Func<UnixFileSystemInfo, bool>)method2.CreateDelegate(typeof(Func<UnixFileSystemInfo, bool>));
         }
 
         public static string GetOriginalPath(this UnixFileSystemInfo info) => GetOriginalPathFunc(info);
 
-        public static long GetSizeOnDisk(this UnixFileSystemInfo info) => info.BlockSize * info.BlocksAllocated;
+        public static bool GetValid(this UnixFileSystemInfo info) => GetValidFunc(info);
+
+        public static long GetSizeOnDisk(this UnixFileSystemInfo info) => info.BlocksAllocated * 512;
     }
 }

Files/UnixFileRecord.cs

@@ -0,0 +1,224 @@
+using System;
+using System.Buffers;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using Mono.Unix;
+
+namespace Files
+{
+    public static class UnixFileRecordExtensions
+    {
+        public static void CalculateHashes(this IEnumerable<UnixFileRecord> records)
+        {
+            foreach (var recordsGroup in records.GroupBy(r => r.INode))
+            {
+                Guid? hash = UnixFileRecord.GetHash(recordsGroup.First().FileInfo);
+                if (!hash.HasValue) continue;
+
+                foreach (UnixFileRecord unixFileRecord in recordsGroup)
+                {
+                    unixFileRecord.SetHash(hash.Value);
+                }
+            }
+        }
+    }
+
+    public class UnixFileRecord
+    {
+        private readonly Lazy<Guid?> _guid;
+        private readonly Lazy<UnixFileInfo> _fileInfo;
+        private Guid? _preCalculatedHash = null;
+
+        public UnixFileRecord()
+        {
+            _guid = new Lazy<Guid?>(GetHash);
+            _fileInfo = new Lazy<UnixFileInfo>(GetFileInfo);
+        }
+
+        public UnixFileRecord(string filePath, long size, long iNode)
+        {
+            Name = filePath;
+            Size = size;
+            INode = iNode;
+            _guid = new Lazy<Guid?>(GetHash);
+            _fileInfo = new Lazy<UnixFileInfo>(GetFileInfo);
+        }
+
+        public UnixFileRecord(UnixFileInfo fileInfo)
+        {
+            _guid = new Lazy<Guid?>(GetHash);
+            _fileInfo = new Lazy<UnixFileInfo>(fileInfo);
+            Name = fileInfo.GetOriginalPath();
+            Size = fileInfo.Length;
+            INode = fileInfo.Inode;
+        }
+
+        public UnixFileRecord(UnixFileInfo fileInfo, Guid hash)
+        {
+            _guid = new Lazy<Guid?>(hash);
+            _fileInfo = new Lazy<UnixFileInfo>(fileInfo);
+            Name = fileInfo.GetOriginalPath();
+            Size = fileInfo.Length;
+            INode = fileInfo.Inode;
+        }
+
+        public string Name { get; init; }
+        public long Size { get; init; }
+        public long INode { get; init; }
+
+        public Guid? Hash => _guid.Value;
+        public UnixFileInfo FileInfo => _fileInfo.Value;
+
+        public void SetHash(Guid hash) => _preCalculatedHash = hash;
+
+        private UnixFileInfo GetFileInfo() => new(Name);
+
+        private Guid? GetHash() => _preCalculatedHash ??= GetHash(Name);
+
+        private async Task<Guid?> GetHashAsync(CancellationToken ct = default) => await GetHashAsync(Name, ct);
+
+        private Guid? GetHash2(CancellationToken ct = default) => GetHash2(Name, ct);
+
+        private async Task<Guid?> GetHash2Async(CancellationToken ct = default) => await GetHash2Async(Name, ct);
+
+        public static Guid? GetHash(string filePath)
+        {
+            try
+            {
+                using FileStream stream = File.OpenRead(filePath);
+                var md5 = MD5.Create();
+                var bytes = md5.ComputeHash(stream);
+                return new Guid(bytes);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public static Guid? GetHash(UnixFileInfo file)
+        {
+            try
+            {
+                using UnixStream stream = file.Open(FileMode.Open);
+                var md5 = MD5.Create();
+                var bytes = md5.ComputeHash(stream);
+                return new Guid(bytes);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public static async Task<Guid?> GetHashAsync(string filePath, CancellationToken ct = default)
+        {
+            try
+            {
+                await using FileStream stream = File.OpenRead(filePath);
+                var md5 = MD5.Create();
+                var bytes = await md5.ComputeHashAsync(stream, ct);
+                return new Guid(bytes);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public static async Task<Guid?> GetHashAsync(UnixFileInfo file, CancellationToken ct = default)
+        {
+            try
+            {
+                await using UnixStream stream = file.Open(FileMode.Open);
+                var md5 = MD5.Create();
+                var bytes = await md5.ComputeHashAsync(stream, ct);
+                return new Guid(bytes);
+            }
+            catch
+            {
+                return null;
+            }
+        }
+
+        public static Guid? GetHash2(string filePath, CancellationToken ct = default)
+        {
+            using IncrementalHash incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.MD5);
+            using FileStream inputStream = File.OpenRead(filePath);
+
+            byte[] buffer = ArrayPool<byte>.Shared.Rent(4096);
+
+            try
+            {
+                int bytesRead;
+                int clearLimit = 0;
+
+                while ((bytesRead = inputStream.Read(buffer, 0, buffer.Length)) > 0)
+                {
+                    if (bytesRead > clearLimit)
+                    {
+                        clearLimit = bytesRead;
+                    }
+
+                    if (ct.IsCancellationRequested) return null;
+
+                    incrementalHash.AppendData(buffer, 0, bytesRead);
+                }
+
+                byte[] hashBytes = incrementalHash.GetHashAndReset();
+                return new Guid(hashBytes);
+            }
+            catch
+            {
+                return null;
+            }
+            finally
+            {
+                //CryptographicOperations.ZeroMemory(buffer.AsSpan(0, clearLimit));
+                ArrayPool<byte>.Shared.Return(buffer, clearArray: false);
+            }
+        }
+
+        public static async Task<Guid?> GetHash2Async(string filePath, CancellationToken ct = default)
+        {
+            using IncrementalHash incrementalHash = IncrementalHash.CreateHash(HashAlgorithmName.MD5);
+            await using FileStream inputStream = File.OpenRead(filePath);
+
+            byte[] buffer = ArrayPool<byte>.Shared.Rent(4096);
+
+            try
+            {
+                int bytesRead;
+                int clearLimit = 0;
+
+                while ((bytesRead = await inputStream.ReadAsync(buffer, 0, buffer.Length, ct)) > 0)
+                {
+                    if (bytesRead > clearLimit)
+                    {
+                        clearLimit = bytesRead;
+                    }
+
+                    if (ct.IsCancellationRequested) return null;
+
+                    incrementalHash.AppendData(buffer, 0, bytesRead);
+                }
+
+                byte[] hashBytes = incrementalHash.GetHashAndReset();
+                return new Guid(hashBytes);
+            }
+            catch
+            {
+                return null;
+            }
+            finally
+            {
+                //CryptographicOperations.ZeroMemory(buffer.AsSpan(0, clearLimit));
+                ArrayPool<byte>.Shared.Return(buffer, clearArray: false);
+            }
+        }
+    }
+}

Files/UnixFileSystemEnumerator.cs

@@ -0,0 +1,247 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Runtime.CompilerServices;
+using System.Threading;
+using Mono.Unix;
+using Mono.Unix.Native;
+
+namespace Files
+{
+    [Flags]
+    public enum SearchTarget : byte
+    {
+        /// <summary>
+        /// Directories.
+        /// </summary>
+        Directories = 1,
+
+        /// <summary>
+        /// Regular files.
+        /// </summary>
+        Files = 2,
+
+        /// <summary>
+        /// Symbolic links.
+        /// </summary>
+        SymLinks = 4,
+
+        /// <summary>
+        /// Named pipes, or FIFOs.
+        /// </summary>
+        NamedPipes = 8,
+
+        /// <summary>
+        /// Local-domain socket.
+        /// </summary>
+        Sockets = 16,
+
+        /// <summary>
+        /// Character devices.
+        /// </summary>
+        CharacterDevices = 32,
+
+        /// <summary>
+        /// Block devices.
+        /// </summary>
+        BlockDevices = 64,
+
+        DirectoriesAndFiles = Directories | Files,
+        DirectoriesAndFilesAndSymLinks = Directories | Files | SymLinks,
+    }
+
+    /* File types for `d_type'.  */
+    public static class DirentType
+    {
+        /// <summary>
+        /// The type is unknown. Only some filesystems have full support to return the type of the file, others might always return this value.
+        /// </summary>
+        public const byte DT_UNKNOWN = 0;
+
+        /// <summary>
+        /// A named pipe, or FIFO.
+        /// </summary>
+        public const byte DT_FIFO = 1;
+
+        /// <summary>
+        /// A character device.
+        /// </summary>
+        public const byte DT_CHR = 2;
+
+        /// <summary>
+        /// A directory.
+        /// </summary>
+        public const byte DT_DIR = 4;
+
+        /// <summary>
+        /// A block device.
+        /// </summary>
+        public const byte DT_BLK = 6;
+
+        /// <summary>
+        /// A regular file.
+        /// </summary>
+        public const byte DT_REG = 8;
+
+        /// <summary>
+        /// A symbolic link.
+        /// </summary>
+        public const byte DT_LNK = 10;
+
+        /// <summary>
+        /// A local-domain socket.
+        /// </summary>
+        public const byte DT_SOCK = 12;
+    }
+
+    public static class UnixFileSystemEnumerator
+    {
+        private const string RelativeCurrentDir = ".";
+        private const string RelativeParentDir = "..";
+
+        public static bool TryGetEntries(string directoryPath, out List<Dirent> list, out Errno error)
+        {
+            list = new List<Dirent>();
+            IntPtr dirPointer = Syscall.opendir(directoryPath);
+            if (dirPointer == IntPtr.Zero)
+            {
+                error = Stdlib.GetLastError();
+                return false;
+            }
+
+            IntPtr result;
+            int returnValue;
+            do
+            {
+                Dirent entry = new();
+                returnValue = Syscall.readdir_r(dirPointer, entry, out result);
+
+                if (returnValue == 0 && result != IntPtr.Zero && (entry.d_name != RelativeCurrentDir && entry.d_name != RelativeParentDir))
+                {
+                    list.Add(entry);
+                }
+            }
+            while (returnValue == 0 && result != IntPtr.Zero);
+
+            if (returnValue == 0)
+            {
+                error = 0;
+                return true;
+            }
+
+            error = Stdlib.GetLastError();
+            return false;
+        }
+
+        public delegate bool FilterEnumeratorDelegate(string directory, string directoryEntry, byte entryType,
+            Errno errno);
+
+        public static IEnumerable<(string path, byte type, Errno errno)> EnumeratePaths(
+            string path, FilterEnumeratorDelegate filter = null, CancellationToken ct = default)
+        {
+            LinkedList<string> directoriesStack = new LinkedList<string>();
+            directoriesStack.AddLast(path);
+
+            while (directoriesStack.Last != null)
+            {
+                string dir = directoriesStack.Last.ValueRef;
+                directoriesStack.RemoveLast();
+
+                if (!TryGetEntries(dir, out List<Dirent> entries, out Errno errno))
+                {
+                    if(!(filter?.Invoke(dir, ".", DirentType.DT_DIR, errno) ?? true)) yield break;
+                    yield return (dir, DirentType.DT_DIR, errno);
+                }
+
+                foreach (Dirent entry in entries)
+                {
+                    ct.ThrowIfCancellationRequested();
+
+                    if (!(filter?.Invoke(dir, entry.d_name, entry.d_type, 0) ?? true)) continue;
+
+                    string combinedPath = Path.Combine(dir, entry.d_name);
+
+                    if (entry.d_type == DirentType.DT_DIR) // Directory
+                    {
+                        directoriesStack.AddLast(combinedPath);
+                    }
+
+                    yield return (combinedPath, entry.d_type, 0);
+                }
+            }
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static bool IsOfTarget(this Dirent entry, SearchTarget desiredTarget)
+        {
+            return entry.d_type switch
+            {
+                DirentType.DT_DIR => (desiredTarget & SearchTarget.Directories) == SearchTarget.Directories,
+                DirentType.DT_REG => (desiredTarget & SearchTarget.Files) == SearchTarget.Files,
+                DirentType.DT_LNK => (desiredTarget & SearchTarget.SymLinks) == SearchTarget.SymLinks,
+                DirentType.DT_FIFO => (desiredTarget & SearchTarget.NamedPipes) == SearchTarget.NamedPipes,
+                DirentType.DT_SOCK => (desiredTarget & SearchTarget.Sockets) == SearchTarget.Sockets,
+                DirentType.DT_CHR => (desiredTarget & SearchTarget.CharacterDevices) == SearchTarget.CharacterDevices,
+                DirentType.DT_BLK => (desiredTarget & SearchTarget.BlockDevices) == SearchTarget.BlockDevices,
+                _ => false
+            };
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public static bool IsOfTarget(byte entryType, SearchTarget desiredTarget)
+        {
+            return entryType switch
+            {
+                DirentType.DT_DIR => (desiredTarget & SearchTarget.Directories) == SearchTarget.Directories,
+                DirentType.DT_REG => (desiredTarget & SearchTarget.Files) == SearchTarget.Files,
+                DirentType.DT_LNK => (desiredTarget & SearchTarget.SymLinks) == SearchTarget.SymLinks,
+                DirentType.DT_FIFO => (desiredTarget & SearchTarget.NamedPipes) == SearchTarget.NamedPipes,
+                DirentType.DT_SOCK => (desiredTarget & SearchTarget.Sockets) == SearchTarget.Sockets,
+                DirentType.DT_CHR => (desiredTarget & SearchTarget.CharacterDevices) == SearchTarget.CharacterDevices,
+                DirentType.DT_BLK => (desiredTarget & SearchTarget.BlockDevices) == SearchTarget.BlockDevices,
+                _ => false
+            };
+        }
+
+        public static Exception CreateExceptionForError(this Errno errno)
+        {
+            string errorDescription = UnixMarshal.GetErrorDescription(errno);
+            UnixIOException unixIoException = new(errno);
+            switch (errno)
+            {
+                case Errno.EPERM:
+                case Errno.EOPNOTSUPP:
+                    return new InvalidOperationException(errorDescription, unixIoException);
+                case Errno.ENOENT:
+                    return new FileNotFoundException(errorDescription, unixIoException);
+                case Errno.EIO:
+                case Errno.ENXIO:
+                case Errno.ENOSPC:
+                case Errno.ESPIPE:
+                case Errno.EROFS:
+                case Errno.ENOTEMPTY:
+                    return new IOException(errorDescription, unixIoException);
+                case Errno.ENOEXEC:
+                    return new InvalidProgramException(errorDescription, unixIoException);
+                case Errno.EBADF:
+                case Errno.EINVAL:
+                    return new ArgumentException(errorDescription, unixIoException);
+                case Errno.EACCES:
+                case Errno.EISDIR:
+                    return new UnauthorizedAccessException(errorDescription, unixIoException);
+                case Errno.EFAULT:
+                    return new NullReferenceException(errorDescription, unixIoException);
+                case Errno.ENOTDIR:
+                    return new DirectoryNotFoundException(errorDescription, unixIoException);
+                case Errno.ERANGE:
+                    return new ArgumentOutOfRangeException(errorDescription);
+                case Errno.ENAMETOOLONG:
+                    return new PathTooLongException(errorDescription, unixIoException);
+                case Errno.EOVERFLOW:
+                    return new OverflowException(errorDescription, unixIoException);
+                default:
+                    return unixIoException;
+            }
+        }
+    }
+}