feat: canister logs extensions

docs/guides/canister-management/settings.mdx

@@ -197,8 +197,6 @@ settings:
 
 ### Log memory limit
 
-{/* Needs human verification: log_memory_limit is exposed by icp-cli but is absent from the canonical ic.did: verify whether this is a management canister setting or an icp-cli layer setting */}
-
 Maximum memory for storing canister logs. Oldest logs are purged when usage exceeds this value.
 
 | Property | Value |

docs/references/ic-interface-spec/abstract-behavior.md

@@ -515,6 +515,7 @@ S = {
   certified_data: CanisterId ↦ Blob;
   canister_history: CanisterId ↦ CanisterHistory;
   canister_log_visibility: CanisterId ↦ CanisterLogVisibility;
+  canister_log_memory_limit: CanisterId ↦ Nat;
   canister_snapshot_visibility: CanisterId ↦ CanisterSnapshotVisibility;
   canister_logs: CanisterId ↦ [CanisterLog];
   query_stats: CanisterId ↦ [QueryStats];
@@ -624,6 +625,7 @@ The initial state of the IC is
   certified_data = ();
   canister_history = ();
   canister_log_visibility = ();
+  canister_log_memory_limit = ();
   canister_snapshot_visibility = ();
   canister_logs = ();
   query_stats = ();
@@ -1704,6 +1706,11 @@ if A.settings.log_visibility is not null:
 else:
   New_canister_log_visibility = Controllers
 
+if A.settings.log_memory_limit is not null:
+  New_canister_log_memory_limit = A.settings.log_memory_limit
+else:
+  New_canister_log_memory_limit = 4096
+
 if A.settings.snapshot_visibility is not null:
   New_canister_snapshot_visibility = A.settings.snapshot_visibility
 else:
@@ -1735,6 +1742,7 @@ S' = S with
     query_stats[Canister_id] = []
     canister_history[Canister_id] = New_canister_history
     canister_log_visibility[Canister_id] = New_canister_log_visibility
+    canister_log_memory_limit[Canister_id] = New_canister_log_memory_limit
     canister_snapshot_visibility[Canister_id] = New_canister_snapshot_visibility
     canister_logs[Canister_id] = []
     messages = Older_messages · Younger_messages ·
@@ -1871,6 +1879,8 @@ S' = S with
     canister_version[A.canister_id] = S.canister_version[A.canister_id] + 1
     if A.settings.log_visibility is not null:
       canister_log_visibility[A.canister_id] = A.settings.log_visibility
+    if A.settings.log_memory_limit is not null:
+      canister_log_memory_limit[A.canister_id] = A.settings.log_memory_limit
     if A.settings.snapshot_visibility is not null:
       canister_snapshot_visibility[A.canister_id] = A.settings.snapshot_visibility
     messages = Older_messages · Younger_messages ·
@@ -2896,7 +2906,8 @@ S with
     certified_data[A.canister_id] = (deleted)
     canister_history[A.canister_id] = (deleted)
     canister_log_visibility[A.canister_id] = (deleted)
-   canister_snapshot_visibility[A.canister_id] = (deleted)
+    canister_log_memory_limit[A.canister_id] = (deleted)
+    canister_snapshot_visibility[A.canister_id] = (deleted)
     canister_logs[A.canister_id] = (deleted)
     query_stats[A.canister_id] = (deleted)
     chunk_store[A.canister_id] = (deleted)
@@ -3129,6 +3140,11 @@ if A.settings.log_visibility is not null:
 else:
   New_canister_log_visibility = Controllers
 
+if A.settings.log_memory_limit is not null:
+  New_canister_log_memory_limit = A.settings.log_memory_limit
+else:
+  New_canister_log_memory_limit = 4096
+
 if A.settings.snapshot_visibility is not null:
   New_canister_snapshot_visibility = A.settings.snapshot_visibility
 else:
@@ -3158,6 +3174,7 @@ S' = S with
     certified_data[Canister_id] = ""
     canister_history[Canister_id] = New_canister_history
     canister_log_visibility[Canister_id] = New_canister_log_visibility
+    canister_log_memory_limit[Canister_id] = New_canister_log_memory_limit
     canister_snapshot_visibility[Canister_id] = New_canister_snapshot_visibility
     canister_logs[Canister_id] = []
     query_stats[CanisterId] = []
@@ -3799,6 +3816,101 @@ S with
 
 ```
 
+#### IC Management Canister: Canister logs {#ic-mgmt-canister-fetch-canister-logs}
+
+Given a state `S`, `Canister_id`, `filter`, and `Sender`, we define
+
+```html
+
+canister_logs(S, Canister_id, filter) =
+  if filter = by_idx Range:
+    { Log | Log ∈ S.canister_logs[Canister_id] ∧ Range.start <= Log.idx ∧ Log.idx < Range.end }
+  else if filter = by_timestamp_nanos Range:
+    { Log | Log ∈ S.canister_logs[Canister_id] ∧ Range.start <= Log.timestamp_nanos ∧ Log.timestamp_nanos < Range.end }
+  else:
+    S.canister_logs[Canister_id]
+fetch_canister_logs_cost(S, Canister_id) = <implementation-specific>
+is_sender_authorized(S, Canister_id, Sender) =
+  (S[Canister_id].canister_log_visibility = Public)
+  or
+  (S[Canister_id].canister_log_visibility = Controllers and Sender in S[Canister_id].controllers)
+  or
+  (S[Canister_id].canister_log_visibility = AllowedViewers Principals and (Sender in S[Canister_id].controllers or Sender in Principals))
+
+```
+
+Conditions
+
+```html
+
+S.messages = Older_messages · CallMessage M · Younger_messages
+(M.queue = Unordered) or (∀ CallMessage M' | FuncMessage M' ∈ Older_messages. M'.queue ≠ M.queue)
+M.callee = ic_principal
+M.method_name = 'fetch_canister_logs'
+M.arg = candid(A)
+fetch_canister_logs_cost(S, A.canister_id) <= M.transferred_cycles
+is_sender_authorized(S, A.canister_id, M.caller)
+
+```
+
+State after
+
+```html
+
+S with
+    messages = Older_messages · Younger_messages ·
+      ResponseMessage {
+        origin = M.origin
+        response = candid(canister_logs(S, A.canister_id, A.filter))
+        refunded_cycles = M.transferred_cycles - fetch_canister_logs_cost(S, A.canister_id)
+      }
+
+```
+
+The IC method `fetch_canister_logs` can also be invoked via management canister query calls.
+They are calls to `/api/v3/canister/<ECID>/query`
+with CBOR content `Q` such that `Q.canister_id = ic_principal`.
+
+Submitted request to `/api/v3/canister/<ECID>/query`
+
+```html
+
+E : Envelope
+
+```
+
+Conditions
+
+```html
+
+E.content = CanisterQuery Q
+Q.canister_id = ic_principal
+Q.method_name = 'fetch_canister_logs'
+|Q.nonce| <= 32
+is_effective_canister_id(E.content, ECID)
+S.system_time <= Q.ingress_expiry or Q.sender = anonymous_id
+Q.arg = candid(A)
+A.canister_id ∈ verify_envelope(E, Q.sender, S.system_time)
+is_sender_authorized(S, A.canister_id, Q.sender)
+
+```
+
+Query response `R`:
+
+```html
+
+{status: "replied"; reply: {arg: candid(canister_logs(S, A.canister_id, A.filter))}, signatures: Sigs}
+
+```
+
+where the query `Q`, the response `R`, and a certificate `Cert` that is obtained by requesting the path `/subnet` in a **separate** read state request to `/api/v3/canister/<ECID>/read_state` satisfy the following:
+
+```html
+
+verify_response(Q, R, Cert) ∧ lookup(["time"], Cert) = Found S.system_time // or "recent enough"
+
+```
+
 #### Callback invocation
 
 When an inter-canister call has been responded to, we can queue the call to the callback.
@@ -4137,6 +4249,8 @@ S with
   canister_history[Canister_id] = (deleted)
   canister_log_visibility[New_canister_id] = S.canister_log_visibility[Canister_id]
   canister_log_visibility[Canister_id] = (deleted)
+  canister_log_memory_limit[New_canister_id] = S.canister_log_memory_limit[Canister_id]
+  canister_log_memory_limit[Canister_id] = (deleted)
   canister_snapshot_visibility[New_canister_id] = S.canister_snapshot_visibility[Canister_id]
   canister_snapshot_visibility[Canister_id] = (deleted)
   canister_logs[New_canister_id] = S.canister_logs[Canister_id]
@@ -4300,16 +4414,19 @@ S with
 
 ```
 
-#### Trimming canister logs
+#### Purging canister logs
+
+Oldest canister logs are purged if the total memory used for canister logs exceeds the value `log_memory_limit` in canister settings.
 
-Canister logs can be trimmed if their total length exceeds 4KiB.
+The (unspecified) function `canister_log_memory_usage(logs)` models the total memory used by `logs`.
 
 Conditions
 
 ```html
 
 S.canister_logs[CanisterId] = Older_logs · Newer_logs
-SUM { |l| | l <- Older_logs } > 4KiB
+canister_log_memory_usage(Older_logs · Newer_logs) > S.canister_log_memory_limit[CanisterId]
+canister_log_memory_usage(Newer_logs) ≤ S.canister_log_memory_limit[CanisterId]
 
 ```
 

docs/references/ic-interface-spec/management-canister.md

@@ -99,6 +99,13 @@ The optional `settings` parameter can be used to set the following settings:
 
     Default value: `controllers`.
 
+-   `log_memory_limit` (`nat`)
+
+    Must be at most `2097152` (`2 MiB`) and indicates the maximum amount of memory used for canister logs.
+    Oldest canister logs are purged if the total memory used for canister logs exceeds this value.
+
+    Default value: `4096`.
+
 -   `snapshot_visibility` (`snapshot_visibility`)
 
     Controls who can access the canister's snapshots through the following endpoints of the management canister:
@@ -304,6 +311,8 @@ Only the controllers of the canister or the canister itself or subnet admins can
 
     * `snapshots_size`: Represents the memory consumed by all snapshots that belong to this canister.
 
+    * `log_memory_store_size`: Represents the memory used by canister logs of the canister.
+
 All sizes are expressed in bytes.
 
 ### IC method `canister_metrics` {#ic-canister_metrics}
@@ -921,12 +930,12 @@ A snapshot may be deleted only by the controllers of the canister that the snaps
 
 ### IC method `fetch_canister_logs` {#ic-fetch_canister_logs}
 
-This method can only be called by external users via non-replicated (query) calls, i.e., it cannot be called by canisters, cannot be called via replicated calls, and cannot be called from composite query calls.
+This method can only be called by external users via non-replicated (query) calls or by canisters (via replicated calls), i.e., it cannot be called by external users via replicated (update) calls and it cannot be called from composite query calls.
 
 Given a canister ID as input, this method returns a vector of logs of that canister including its trap messages.
 The canister logs are *not* collected in canister methods running in non-replicated mode (NRQ, TQ, CQ, CRy, CRt, CC, and F modes, as defined in [Overview of imports](./canister-interface.md#system-api-imports)) and the canister logs are *purged* when the canister is reinstalled or uninstalled.
-The total size of all returned logs does not exceed 4KiB.
-If new logs are added resulting in exceeding the maximum total log size of 4KiB, the oldest logs will be removed.
+The total size of all returned logs does not exceed the value `log_memory_limit` in canister settings.
+Oldest canister logs are purged if the total memory used for canister logs exceeds the value `log_memory_limit` in canister settings.
 Logs persist across canister upgrades and they are deleted if the canister is reinstalled or uninstalled.
 
 The log visibility is defined in the `log_visibility` field of `canister_settings` and can be one of the following variants:
@@ -941,6 +950,12 @@ A single log is a record with the following fields:
 - `timestamp_nanos` (`nat64`): the timestamp as nanoseconds since 1970-01-01 at which the log was recorded;
 - `content` (`blob`): the actual content of the log;
 
+To filter canister logs, an optional filter can be provided and have one of the following variants:
+- `by_idx` (`record { start : nat64; end : nat64 }`): only logs are returned whose `idx` is within the provided range (`start` is inclusive, but `end` is exclusive);
+- `by_timestamp_nanos` (`record { start : nat64; end : nat64 }`): only logs are returned whose `timestamp_nanos` is within the provided range (`start` is inclusive, but `end` is exclusive).
+
+Cycles to pay for the call must be explicitly transferred with the call, i.e., they are not automatically deducted from the caller's balance implicitly (e.g., as for inter-canister calls).
+
 :::warning
 
 The response of a query comes from a single replica, and is therefore not appropriate for security-sensitive applications.

public/references/ic.did

@@ -26,6 +26,7 @@ type canister_settings = record {
     freezing_threshold : opt nat;
     reserved_cycles_limit : opt nat;
     log_visibility : opt log_visibility;
+    log_memory_limit : opt nat;
     snapshot_visibility : opt snapshot_visibility;
     wasm_memory_limit : opt nat;
     wasm_memory_threshold : opt nat;
@@ -39,6 +40,7 @@ type definite_canister_settings = record {
     freezing_threshold : nat;
     reserved_cycles_limit : nat;
     log_visibility : log_visibility;
+    log_memory_limit : nat;
     snapshot_visibility : snapshot_visibility;
     wasm_memory_limit : nat;
     wasm_memory_threshold : nat;
@@ -300,6 +302,7 @@ type canister_status_result = record {
         canister_history_size : nat;
         wasm_chunk_store_size : nat;
         snapshots_size : nat;
+        log_memory_store_size : nat;
     };
     cycles : nat;
     reserved_cycles : nat;
@@ -499,6 +502,10 @@ type delete_canister_snapshot_args = record {
 
 type fetch_canister_logs_args = record {
     canister_id : canister_id;
+    filter : opt variant {
+        by_idx : record { start : nat64; end : nat64 };
+        by_timestamp_nanos : record { start : nat64; end : nat64 };
+    }
 };
 
 type canister_log_record = record {