using System.Text; using Omika.Core.Observability.External; using Omika.Diagnostics.Data; using Omika.Diagnostics.Import; using Omika.Diagnostics.Tests.Fakes; namespace Omika.Diagnostics.Tests; /// /// §5C.1C — Lecture sûre des lignes JSONL : une ligne finale SANS newline n'est /// jamais retournée ni checkpointée (relue à la passe suivante). Une ligne poison /// COMPLÈTE (terminée par newline) reste sautée et fait avancer le curseur. /// public class JsonlCompleteLineReaderTests { private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false); // ---- Splitteur pur (LF / CRLF / fragment) ------------------------------ [Fact] public void SplitCompleteLines_WithLf_ReturnsAllLines() { byte[] bytes = Utf8NoBom.GetBytes("a\nb\nc\n"); string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length); Assert.Equal(new[] { "a", "b", "c" }, lines); } [Fact] public void SplitCompleteLines_WithCrLf_ReturnsLinesWithoutCr() { byte[] bytes = Utf8NoBom.GetBytes("a\r\nb\r\n"); string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length); Assert.Equal(new[] { "a", "b" }, lines); } [Fact] public void SplitCompleteLines_TrailingFragment_IsExcluded() { byte[] bytes = Utf8NoBom.GetBytes("a\nb\npartial"); string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length); Assert.Equal(new[] { "a", "b" }, lines); // "partial" sans newline est exclu } // ---- Reader : lignes complètes LF / CRLF ------------------------------- [Fact] public void CompleteLine_WithLf_IsRead() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } [Fact] public void CompleteLine_WithCrLf_IsRead() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\r\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } // ---- Ligne finale partielle ------------------------------------------- [Fact] public void PartialFinalLine_IsNotReturned() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // r2 sans newline var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId); // r2 (fragment) non retourné } finally { Directory.Delete(dir, recursive: true); } } [Fact] public void PartialFinalLine_DoesNotAdvanceCheckpoint() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // fragment var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Equal(0, read.NextPosition!.LineIndex); // seul r1 (index 0) est checkpointé } finally { Directory.Delete(dir, recursive: true); } } [Fact] public void PartialLineCompletedLater_IsReadOnNextPass() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // fragment r2 var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult first = source.ReadNext(null, 10); Assert.Single(first.Records); source.Commit(first.NextPosition!); AppendRaw(file, "\n"); // complète le fragment DiagnosticImportCheckpoint? checkpoint = source.LoadCheckpoint(); DiagnosticJsonlReadResult second = source.ReadNext(checkpoint, 10); Assert.Single(second.Records); Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } // ---- Ligne poison complète --------------------------------------------- [Fact] public void CompletePoisonLine_IsSkippedAndCheckpointAdvances() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, "{ json invalide\n" + DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); Assert.Equal(1, read.NextPosition!.LineIndex); // poison (0) sautée, r1 (1) } finally { Directory.Delete(dir, recursive: true); } } // ---- Fichier qui grandit ---------------------------------------------- [Fact] public void GrowingFile_NewCompleteLineReadNextPass() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult first = source.ReadNext(null, 10); source.Commit(first.NextPosition!); AppendRaw(file, DiagnosticTestData.SerializeLine(r2) + "\n"); DiagnosticJsonlReadResult second = source.ReadNext(source.LoadCheckpoint(), 10); Assert.Single(second.Records); Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } // ---- Rotation / fichier disparu ---------------------------------------- [Fact] public void RotatedFileDuringRead_NoPermanentFailure() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B"); string file1 = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); string file2 = Path.Combine(dir, "omika-mobile-diagnostic-20260101-002.jsonl"); WriteRaw(file1, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult first = source.ReadNext(null, 10); source.Commit(first.NextPosition!); File.Delete(file1); // rotation : 001 supprimé WriteRaw(file2, DiagnosticTestData.SerializeLine(r2) + "\n"); DiagnosticJsonlReadResult second = source.ReadNext(source.LoadCheckpoint(), 10); Assert.Single(second.Records); Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } [Fact] public void MissingFile_NoPermanentFailure() { string dir = CreateTempDir(); try { var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); // aucun fichier Assert.Empty(read.Records); Assert.True(read.ReachedEnd); } finally { Directory.Delete(dir, recursive: true); } } // ---- Checkpoint corrompu ---------------------------------------------- [Fact] public void CorruptCheckpoint_FailsSafe() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n"); WriteRaw(Path.Combine(dir, "import-checkpoint.json"), "{ pas du json valide"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); Assert.Null(source.LoadCheckpoint()); // fail-safe → replay DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); } finally { Directory.Delete(dir, recursive: true); } } // ---- Checkpoint séparé de la source ------------------------------------ [Fact] public void CheckpointDirectory_CanDifferFromSource() { string srcDir = CreateTempDir(); string cpDir = Path.Combine(Path.GetTempPath(), "omika-cp-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(cpDir); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(srcDir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(srcDir, "import-checkpoint.json", cpDir); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); source.Commit(read.NextPosition!); Assert.True(File.Exists(Path.Combine(cpDir, "import-checkpoint.json"))); Assert.False(File.Exists(Path.Combine(srcDir, "import-checkpoint.json"))); } finally { Directory.Delete(srcDir, recursive: true); Directory.Delete(cpDir, recursive: true); } } [Fact] public void CheckpointDirectory_DefaultCompatibility() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); // pas de checkpointDir DiagnosticJsonlReadResult read = source.ReadNext(null, 10); source.Commit(read.NextPosition!); Assert.True(File.Exists(Path.Combine(dir, "import-checkpoint.json"))); } finally { Directory.Delete(dir, recursive: true); } } // ---- Staging *.copying ------------------------------------------------- [Fact] public void CopyingFile_IsIgnored() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string copying = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl.copying"); WriteRaw(copying, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Empty(read.Records); Assert.True(read.ReachedEnd); } finally { Directory.Delete(dir, recursive: true); } } [Fact] public void CopyingRenamedToJsonl_IsImported() { string dir = CreateTempDir(); try { RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A"); string copying = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl.copying"); string final = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl"); WriteRaw(copying, DiagnosticTestData.SerializeLine(r1) + "\n"); var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); Assert.Empty(source.ReadNext(null, 10).Records); // .copying ignoré File.Move(copying, final); // rename atomique DiagnosticJsonlReadResult read = source.ReadNext(null, 10); Assert.Single(read.Records); Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId); } finally { Directory.Delete(dir, recursive: true); } } // ---- Helpers ----------------------------------------------------------- private static void WriteRaw(string path, string content) => File.WriteAllText(path, content, Utf8NoBom); private static void AppendRaw(string path, string content) => File.AppendAllText(path, content, Utf8NoBom); private static string CreateTempDir() { string dir = Path.Combine(Path.GetTempPath(), "omika-diag-test-" + Guid.NewGuid().ToString("N")); Directory.CreateDirectory(dir); return dir; } }