Premiere commit - Omika
This commit is contained in:
@@ -0,0 +1,366 @@
|
||||
using System.Text;
|
||||
using Omika.Core.Observability.External;
|
||||
using Omika.Diagnostics.Data;
|
||||
using Omika.Diagnostics.Import;
|
||||
using Omika.Diagnostics.Tests.Fakes;
|
||||
|
||||
namespace Omika.Diagnostics.Tests;
|
||||
|
||||
/// <summary>
|
||||
/// §5C.1C — Lecture sûre des lignes JSONL : une ligne finale SANS newline n'est
|
||||
/// jamais retournée ni checkpointée (relue à la passe suivante). Une ligne poison
|
||||
/// COMPLÈTE (terminée par newline) reste sautée et fait avancer le curseur.
|
||||
/// </summary>
|
||||
public class JsonlCompleteLineReaderTests
|
||||
{
|
||||
private static readonly UTF8Encoding Utf8NoBom = new(encoderShouldEmitUTF8Identifier: false);
|
||||
|
||||
// ---- Splitteur pur (LF / CRLF / fragment) ------------------------------
|
||||
|
||||
[Fact]
|
||||
public void SplitCompleteLines_WithLf_ReturnsAllLines()
|
||||
{
|
||||
byte[] bytes = Utf8NoBom.GetBytes("a\nb\nc\n");
|
||||
string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length);
|
||||
Assert.Equal(new[] { "a", "b", "c" }, lines);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SplitCompleteLines_WithCrLf_ReturnsLinesWithoutCr()
|
||||
{
|
||||
byte[] bytes = Utf8NoBom.GetBytes("a\r\nb\r\n");
|
||||
string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length);
|
||||
Assert.Equal(new[] { "a", "b" }, lines);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void SplitCompleteLines_TrailingFragment_IsExcluded()
|
||||
{
|
||||
byte[] bytes = Utf8NoBom.GetBytes("a\nb\npartial");
|
||||
string[] lines = JsonlCompleteLineReader.SplitCompleteLines(bytes, bytes.Length);
|
||||
Assert.Equal(new[] { "a", "b" }, lines); // "partial" sans newline est exclu
|
||||
}
|
||||
|
||||
// ---- Reader : lignes complètes LF / CRLF -------------------------------
|
||||
|
||||
[Fact]
|
||||
public void CompleteLine_WithLf_IsRead()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Single(read.Records);
|
||||
Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CompleteLine_WithCrLf_IsRead()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\r\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Single(read.Records);
|
||||
Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Ligne finale partielle -------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void PartialFinalLine_IsNotReturned()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // r2 sans newline
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Single(read.Records);
|
||||
Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId); // r2 (fragment) non retourné
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PartialFinalLine_DoesNotAdvanceCheckpoint()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // fragment
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Equal(0, read.NextPosition!.LineIndex); // seul r1 (index 0) est checkpointé
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void PartialLineCompletedLater_IsReadOnNextPass()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n" + DiagnosticTestData.SerializeLine(r2)); // fragment r2
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult first = source.ReadNext(null, 10);
|
||||
Assert.Single(first.Records);
|
||||
source.Commit(first.NextPosition!);
|
||||
|
||||
AppendRaw(file, "\n"); // complète le fragment
|
||||
DiagnosticImportCheckpoint? checkpoint = source.LoadCheckpoint();
|
||||
DiagnosticJsonlReadResult second = source.ReadNext(checkpoint, 10);
|
||||
|
||||
Assert.Single(second.Records);
|
||||
Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Ligne poison complète ---------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void CompletePoisonLine_IsSkippedAndCheckpointAdvances()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, "{ json invalide\n" + DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Single(read.Records);
|
||||
Assert.Equal(1, read.NextPosition!.LineIndex); // poison (0) sautée, r1 (1)
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Fichier qui grandit ----------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void GrowingFile_NewCompleteLineReadNextPass()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult first = source.ReadNext(null, 10);
|
||||
source.Commit(first.NextPosition!);
|
||||
|
||||
AppendRaw(file, DiagnosticTestData.SerializeLine(r2) + "\n");
|
||||
DiagnosticJsonlReadResult second = source.ReadNext(source.LoadCheckpoint(), 10);
|
||||
|
||||
Assert.Single(second.Records);
|
||||
Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Rotation / fichier disparu ----------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void RotatedFileDuringRead_NoPermanentFailure()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
RemoteDiagnosticRecord r2 = DiagnosticTestData.Record(Guid.NewGuid(), "B");
|
||||
string file1 = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
string file2 = Path.Combine(dir, "omika-mobile-diagnostic-20260101-002.jsonl");
|
||||
WriteRaw(file1, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult first = source.ReadNext(null, 10);
|
||||
source.Commit(first.NextPosition!);
|
||||
|
||||
File.Delete(file1); // rotation : 001 supprimé
|
||||
WriteRaw(file2, DiagnosticTestData.SerializeLine(r2) + "\n");
|
||||
|
||||
DiagnosticJsonlReadResult second = source.ReadNext(source.LoadCheckpoint(), 10);
|
||||
Assert.Single(second.Records);
|
||||
Assert.Equal(r2.Event.EventId, second.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void MissingFile_NoPermanentFailure()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10); // aucun fichier
|
||||
Assert.Empty(read.Records);
|
||||
Assert.True(read.ReachedEnd);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Checkpoint corrompu ----------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void CorruptCheckpoint_FailsSafe()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
WriteRaw(Path.Combine(dir, "import-checkpoint.json"), "{ pas du json valide");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
Assert.Null(source.LoadCheckpoint()); // fail-safe → replay
|
||||
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
Assert.Single(read.Records);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Checkpoint séparé de la source ------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void CheckpointDirectory_CanDifferFromSource()
|
||||
{
|
||||
string srcDir = CreateTempDir();
|
||||
string cpDir = Path.Combine(Path.GetTempPath(), "omika-cp-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(cpDir);
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(srcDir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(srcDir, "import-checkpoint.json", cpDir);
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
source.Commit(read.NextPosition!);
|
||||
|
||||
Assert.True(File.Exists(Path.Combine(cpDir, "import-checkpoint.json")));
|
||||
Assert.False(File.Exists(Path.Combine(srcDir, "import-checkpoint.json")));
|
||||
}
|
||||
finally
|
||||
{
|
||||
Directory.Delete(srcDir, recursive: true);
|
||||
Directory.Delete(cpDir, recursive: true);
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CheckpointDirectory_DefaultCompatibility()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string file = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(file, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json"); // pas de checkpointDir
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
source.Commit(read.NextPosition!);
|
||||
|
||||
Assert.True(File.Exists(Path.Combine(dir, "import-checkpoint.json")));
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Staging *.copying -------------------------------------------------
|
||||
|
||||
[Fact]
|
||||
public void CopyingFile_IsIgnored()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string copying = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl.copying");
|
||||
WriteRaw(copying, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
|
||||
Assert.Empty(read.Records);
|
||||
Assert.True(read.ReachedEnd);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void CopyingRenamedToJsonl_IsImported()
|
||||
{
|
||||
string dir = CreateTempDir();
|
||||
try
|
||||
{
|
||||
RemoteDiagnosticRecord r1 = DiagnosticTestData.Record(Guid.NewGuid(), "A");
|
||||
string copying = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl.copying");
|
||||
string final = Path.Combine(dir, "omika-mobile-diagnostic-20260101-001.jsonl");
|
||||
WriteRaw(copying, DiagnosticTestData.SerializeLine(r1) + "\n");
|
||||
|
||||
var source = new JsonlDiagnosticSource(dir, "import-checkpoint.json");
|
||||
Assert.Empty(source.ReadNext(null, 10).Records); // .copying ignoré
|
||||
|
||||
File.Move(copying, final); // rename atomique
|
||||
DiagnosticJsonlReadResult read = source.ReadNext(null, 10);
|
||||
Assert.Single(read.Records);
|
||||
Assert.Equal(r1.Event.EventId, read.Records[0].Event.EventId);
|
||||
}
|
||||
finally { Directory.Delete(dir, recursive: true); }
|
||||
}
|
||||
|
||||
// ---- Helpers -----------------------------------------------------------
|
||||
|
||||
private static void WriteRaw(string path, string content) =>
|
||||
File.WriteAllText(path, content, Utf8NoBom);
|
||||
|
||||
private static void AppendRaw(string path, string content) =>
|
||||
File.AppendAllText(path, content, Utf8NoBom);
|
||||
|
||||
private static string CreateTempDir()
|
||||
{
|
||||
string dir = Path.Combine(Path.GetTempPath(), "omika-diag-test-" + Guid.NewGuid().ToString("N"));
|
||||
Directory.CreateDirectory(dir);
|
||||
return dir;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user