Files/Files/Program.cs

using System;
using System.Collections.Generic;
using System.IO;
using Spectre.Console;
using Microsoft.Data.Sqlite;
using Mono.Unix;
using System.Threading;
using System.CommandLine;
using System.CommandLine.Invocation;
using System.CommandLine.Parsing;
using System.Threading.Tasks;
using System.Linq;
using Dapper;
using System.Security.Cryptography;

namespace Files {
    class Program {
        private static async Task IndexFiles(bool isVerbose, DirectoryInfo startDirectory, CancellationToken ct) {
            await AnsiConsole.Status()
                .StartAsync("Thinking...", async ctx => {
                    using var connection = new SqliteConnection("Data Source=db.db");
                    connection.Open();
                    await using var transaction = await connection.BeginTransactionAsync();

                    var cnt = connection.ExecuteScalar<int>("SELECT count(*) FROM sqlite_master WHERE type='table' AND name=@tableName;", new { tableName = "files" });
                    if (cnt == 0) 
                    {
                        connection.Execute("CREATE TABLE IF NOT EXISTS files (name TEXT, size INTEGER, inode INTEGER);");
                    }

                    Queue<string> directoriesQueue = new Queue<string>();
                    directoriesQueue.Enqueue(startDirectory?.ToString() ?? ".");

                    try {
                        while (directoriesQueue.TryDequeue(out string peekedDir)) {
                            ctx.Status(peekedDir.Replace("[", "[[").Replace("]", "]]"));

                            UnixDirectoryInfo dirInfo = new(peekedDir);
                            if (!dirInfo.CanAccess(Mono.Unix.Native.AccessModes.R_OK)
                            || !dirInfo.CanAccess(Mono.Unix.Native.AccessModes.X_OK)) {
                                AnsiConsole.MarkupLine($"[red]:cross_mark: NO_ACCESS:[/] :file_folder: {dirInfo.ToString().Replace("[", "[[").Replace("]", "]]")}");
                                return;
                            }
                            UnixFileSystemInfo[] entries = dirInfo.GetFileSystemEntries();
                            foreach (UnixFileSystemInfo entry in entries) {
                                string relativePath = Path.Combine(peekedDir, entry.Name);
                                if (!entry.CanAccess(Mono.Unix.Native.AccessModes.R_OK)) {
                                    if (entry.IsDirectory)
                                        AnsiConsole.MarkupLine($"[red]:cross_mark: NO_ACCESS:[/] :file_folder: {relativePath.Replace("[", "[[").Replace("]", "]]")}");
                                    else if (entry.IsRegularFile)
                                        AnsiConsole.MarkupLine($"[red]:cross_mark: NO_ACCESS:[/] :page_facing_up: {relativePath.Replace("[", "[[").Replace("]", "]]")}");
                                    continue;
                                }

                                if (entry.IsDirectory) {
                                    directoriesQueue.Enqueue(relativePath);
                                    continue;
                                }

                                connection.Execute("INSERT INTO files (name, size, inode) VALUES (@name, @Length, @Inode);", new { name = relativePath, entry.Length, entry.Inode });

                                if (isVerbose)
                                    AnsiConsole.MarkupLine($"[green]:check_mark: OK:[/] {relativePath.Replace("[", "[[").Replace("]", "]]")}");

                                if (ct.IsCancellationRequested)
                                    return;
                            }
                        }
                        transaction.Commit();
                    } catch (Exception exception) {
                        await transaction.RollbackAsync();
                        AnsiConsole.WriteException(exception);
                        
                        return;
                    }

                    ctx.Status("Finding duplicates...");
                    ctx.Spinner(Spinner.Known.Aesthetic);

                    var potential = connection.Query<(int cnt, long size)>("SELECT COUNT(*) cnt, size FROM files WHERE size != 0 GROUP BY size HAVING cnt > 1 ORDER BY size * cnt DESC;");

                    foreach (var potentialFile in potential) {
                        var sameSize = connection.Query<DbRecord>("SELECT name, size, inode FROM files WHERE size = @size",
                            new { potentialFile.size }).ToList();

                        var unporocessable = sameSize
                            .Where(r => !r.Hash.HasValue);

                        foreach (var dbRecord in unporocessable)
                        {
                            AnsiConsole.MarkupLine($"[red]:cross_mark: NO_ACCESS:[/] :page_facing_up: {dbRecord.Name.Replace("[", "[[").Replace("]", "]]")}");
                        }

                        var equalGrouped = sameSize
                            .Where(r => r.Hash.HasValue)
                            .GroupBy(r=>r.Hash)
                            .Where(g=>g.Count() > 1)
                            .ToList();

                        foreach (var grp in equalGrouped) {
                            var root = new Tree(":double_exclamation_mark: " + grp.Key);
                            foreach (var item in grp) {
                                root.AddNode(item.Name);
                            }
                            AnsiConsole.Render(root);
                        }
                    }

        });
        }

        private static async Task Main(string[] args) {
            var verboseOption = new Option<bool>(new []{"--verbose", "-v"} ,"Verbose");
            var directoryArgument = new Argument<DirectoryInfo>(
                result => new DirectoryInfo("./"), isDefault: true)
                {
                Name = "directory",
                Description = "Directory to scan.",
                Arity = ArgumentArity.ZeroOrOne,
            }.ExistingOnly();

            var rootCommand = new RootCommand("$ File -v false ./")
            {
                verboseOption,
                directoryArgument,
            };

            ParseResult result = rootCommand.Parse(args);
            ArgumentResult dirResult = result.FindResultFor(directoryArgument);
            var dir = new DirectoryInfo(
                dirResult.Tokens.FirstOrDefault()?.Value 
                ?? dirResult.Argument.GetDefaultValue()?.ToString());

            rootCommand.Handler = CommandHandler.Create<bool, CancellationToken>(
                async (verbose, ct) => await IndexFiles(verbose, dir, ct));

            await rootCommand.InvokeAsync(args);
        }
    }

    public class DbRecord {
        private readonly Lazy<Guid?> _guid;

        public DbRecord() {
            _guid = new Lazy<Guid?>(GetHash);
        }

        public string Name { get; set; }
        public long Size { get; set; }
        public long Inode { get; set; }
        public Guid? Hash => _guid.Value;

        public Guid? GetHash() {
            try {
                using FileStream stream = File.OpenRead(Name);
                var md5 = MD5.Create();
                var bytes = md5.ComputeHash(stream);
                return new Guid(bytes);
            } catch {
                return null;
            }
        }
    }
}