-
Notifications
You must be signed in to change notification settings - Fork 16
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
14 changed files
with
767 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,19 @@ | ||
| { | ||
| "$schema": "http://json.schemastore.org/template", | ||
| "author": "@jet @bartelink", | ||
| "classifications": [ | ||
| "Propulsion", | ||
| "Equinox", | ||
| "CosmosDb", | ||
| "ChangeFeed", | ||
| "ChangeFeedProcessor" | ||
| ], | ||
| "tags": { | ||
| "language": "F#" | ||
| }, | ||
| "identity": "Propulsion.Template.Archive", | ||
| "name": "Propulsion Archiver service", | ||
| "shortName": "proArchiver", | ||
| "sourceName": "ArchiverTemplate", | ||
| "preferNameDirectory": true, | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,25 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <OutputType>Exe</OutputType> | ||
| <TargetFramework>netcoreapp3.1</TargetFramework> | ||
| <WarningLevel>5</WarningLevel> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <Compile Include="Infrastructure.fs" /> | ||
| <Compile Include="Handler.fs" /> | ||
| <Compile Include="Program.fs" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="Argu" Version="6.0.0" /> | ||
| <PackageReference Include="Destructurama.FSharp" Version="1.1.1-dev-00033" /> | ||
| <PackageReference Include="Microsoft.Azure.DocumentDB.ChangeFeedProcessor" Version="2.3.2" /> | ||
| <PackageReference Include="Microsoft.Azure.DocumentDB.Core" Version="2.11.6" /> | ||
| <PackageReference Include="Propulsion.Cosmos" Version="2.9.0" /> | ||
| <PackageReference Include="Serilog.Sinks.Async" Version="1.4.0" /> | ||
| <PackageReference Include="Serilog.Sinks.Console" Version="3.1.1" /> | ||
| <PackageReference Include="Serilog.Sinks.Seq" Version="4.0.0" /> | ||
| </ItemGroup> | ||
| </Project> |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,24 @@ | ||
| module ArchiverTemplate.Handler | ||
|
|
||
| type Stats(log, statsInterval, stateInterval) = | ||
| inherit Propulsion.Streams.Sync.Stats<unit>(log, statsInterval, stateInterval) | ||
|
|
||
| override __.HandleOk(()) = () | ||
| override __.HandleExn(log, exn) = log.Information(exn, "Unhandled") | ||
|
|
||
| let (|Archivable|NotArchivable|) = function | ||
| // TODO define Categories that should be copied to the secondary Container | ||
| | "CategoryName" -> | ||
| Archivable | ||
| | _ -> | ||
| NotArchivable | ||
|
|
||
| let selectArchivable (changeFeedDocument: Microsoft.Azure.Documents.Document) : Propulsion.Streams.StreamEvent<_> seq = seq { | ||
| let s = changeFeedDocument.GetPropertyValue("p") |> string | ||
| if s.StartsWith("events-") then () else | ||
| for batch in Propulsion.Cosmos.EquinoxCosmosParser.enumStreamEvents changeFeedDocument do | ||
| let (FsCodec.StreamName.CategoryAndId (cat,_)) = batch.stream | ||
| match cat with | ||
| | Archivable -> yield batch | ||
| | NotArchivable -> () | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,58 @@ | ||
| [<AutoOpen>] | ||
| module ArchiverTemplate.Infrastructure | ||
|
|
||
| open Serilog | ||
| open System.Runtime.CompilerServices | ||
|
|
||
| [<Extension>] | ||
| type LoggerConfigurationExtensions() = | ||
|
|
||
| [<Extension>] | ||
| static member inline ExcludeChangeFeedProcessorV2InternalDiagnostics(c : LoggerConfiguration) = | ||
| let isCfp429a = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.LeaseManagement.DocumentServiceLeaseUpdater").Invoke | ||
| let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke | ||
| let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke | ||
| let isCfp429d = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing.PartitionProcessor").Invoke | ||
| let isCfp x = isCfp429a x || isCfp429b x || isCfp429c x || isCfp429d x | ||
| c.Filter.ByExcluding(fun x -> isCfp x) | ||
|
|
||
| [<Extension>] | ||
| static member inline ConfigureChangeFeedProcessorLogging(c : LoggerConfiguration, verbose : bool) = | ||
| // LibLog writes to the global logger, so we need to control the emission | ||
| let cfpl = if verbose then Events.LogEventLevel.Debug else Events.LogEventLevel.Warning | ||
| c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpl) | ||
| |> fun c -> if verbose then c else c.ExcludeChangeFeedProcessorV2InternalDiagnostics() | ||
|
|
||
| [<Extension>] | ||
| type Logging() = | ||
|
|
||
| [<Extension>] | ||
| static member Configure(configuration : LoggerConfiguration, verbose, (logSyncToConsole, minRu), cfpVerbose) = | ||
| configuration | ||
| .Destructure.FSharpTypes() | ||
| .Enrich.FromLogContext() | ||
| |> fun c -> if verbose then c.MinimumLevel.Debug() else c | ||
| |> fun c -> c.ConfigureChangeFeedProcessorLogging(cfpVerbose) | ||
| |> fun c -> let ingesterLevel = if logSyncToConsole then Events.LogEventLevel.Debug else Events.LogEventLevel.Information | ||
| c.MinimumLevel.Override(typeof<Propulsion.Streams.Scheduling.StreamSchedulingEngine>.FullName, ingesterLevel) | ||
| |> fun c -> let generalLevel = if verbose then Events.LogEventLevel.Information else Events.LogEventLevel.Warning | ||
| c.MinimumLevel.Override(typeof<Propulsion.Cosmos.Internal.Writer.Result>.FullName, generalLevel) | ||
| |> fun c -> | ||
| let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId,2} {Message:lj} {Properties}{NewLine}{Exception}" | ||
| let t = if verbose then t else t.Replace("{Properties}", "") | ||
| let configure (a : Configuration.LoggerSinkConfiguration) : unit = | ||
| a.Logger(fun l -> | ||
| l.WriteTo.Sink(Equinox.Cosmos.Store.Log.InternalMetrics.Stats.LogSink()) |> ignore) |> ignore | ||
| a.Logger(fun l -> | ||
| let isEqx = Filters.Matching.FromSource<Equinox.Cosmos.Core.Context>().Invoke | ||
| let isWriterB = Filters.Matching.FromSource<Propulsion.Cosmos.Internal.Writer.Result>().Invoke | ||
| let l = if logSyncToConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isWriterB x) | ||
| let isCheaperThan minRu = function | ||
| | Equinox.Cosmos.Store.Log.InternalMetrics.Stats.CosmosMetric | ||
| (Equinox.Cosmos.Store.Log.Event.SyncSuccess m | Equinox.Cosmos.Store.Log.Event.SyncConflict m) -> | ||
| m.ru < minRu | ||
| | _ -> false | ||
| let l = match minRu with Some mru -> l.Filter.ByExcluding(fun x -> isCheaperThan mru x) | None -> l | ||
| l.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) | ||
| |> ignore | ||
| c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=System.Action<_> configure) |
Oops, something went wrong.