diff --git a/docs/screenshots/agent-graph-four-participants-layout-preview.svg b/docs/screenshots/agent-graph-four-participants-layout-preview.svg deleted file mode 100644 index 2a03d844..00000000 --- a/docs/screenshots/agent-graph-four-participants-layout-preview.svg +++ /dev/null @@ -1,189 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 4 participants - current radial layout - Strict small-team preset: top / right / bottom / left around Lead - - - - - - Lead - center reserved zone - - - - - Participant 1 - top side - - slot 1 - - - - - - Participant 2 - right side - - slot 2 - - - - - - Participant 3 - bottom side - - slot 3 - - - - - - Participant 4 - left side - - slot 4 - - \ No newline at end of file diff --git a/docs/screenshots/agent-graph-row-orbit-layout-preview.svg b/docs/screenshots/agent-graph-row-orbit-layout-preview.svg deleted file mode 100644 index 4ffbd6d1..00000000 --- a/docs/screenshots/agent-graph-row-orbit-layout-preview.svg +++ /dev/null @@ -1,125 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 8 participants - 3 top / 2 at lead level / 3 bottom - 12 participants - 4 top / 2 + lead + 2 middle / 4 bottom - - - - - - top row - lead row - bottom row - - - - - - - - - - - - - - - lead - - - alicereviewer - novadeveloper - tomdeveloper - jackdeveloper - atlasassistant - bobdeveloper - mayaqa - kaiops - - - - - - - top row - lead row - bottom row - - - - - - - - - - - - - - - - - - - lead - - - alice - nova - tom - jack - atlas - bob - maya - kai - ivy - rex - zoe - sam - - diff --git a/docs/screenshots/cover-frame.png b/docs/screenshots/cover-frame.png new file mode 100644 index 00000000..5ffef519 Binary files /dev/null and b/docs/screenshots/cover-frame.png differ diff --git a/docs/screenshots/screenshots-animated.webp b/docs/screenshots/screenshots-animated.webp new file mode 100644 index 00000000..000931d5 Binary files /dev/null and b/docs/screenshots/screenshots-animated.webp differ diff --git a/package.json b/package.json index e0e6ac65..e44bcb8f 100644 --- a/package.json +++ b/package.json @@ -21,6 +21,7 @@ }, "main": "dist-electron/main/index.cjs", "scripts": { + "preinstall": "node ./scripts/ci/enforce-pnpm-install.mjs", "dev": "node ./scripts/dev-with-runtime.mjs", "dev:mcp": "node ./scripts/dev-with-runtime.mjs --remoteDebuggingPort 9222", "dev:kill": "node bin/kill-dev.js", @@ -453,6 +454,8 @@ "@radix-ui/react-dismissable-layer@1.1.11": "patches/@radix-ui__react-dismissable-layer@1.1.11.patch", "@radix-ui/react-popper@1.2.8": "patches/@radix-ui__react-popper@1.2.8.patch", "@radix-ui/react-select@2.2.6": "patches/@radix-ui__react-select@2.2.6.patch", + "@radix-ui/react-slot@1.2.3": "patches/@radix-ui__react-slot@1.2.3.patch", + "@radix-ui/react-slot@1.2.4": "patches/@radix-ui__react-slot@1.2.4.patch", "@radix-ui/react-tooltip@1.2.8": "patches/@radix-ui__react-tooltip@1.2.8.patch", "@radix-ui/react-menu@2.1.16": "patches/@radix-ui__react-menu@2.1.16.patch", "@radix-ui/react-checkbox@1.3.3": "patches/@radix-ui__react-checkbox@1.3.3.patch" diff --git a/patches/@radix-ui__react-select@2.2.6.patch b/patches/@radix-ui__react-select@2.2.6.patch index 7c84fd8b..e2dd2022 100644 --- a/patches/@radix-ui__react-select@2.2.6.patch +++ b/patches/@radix-ui__react-select@2.2.6.patch @@ -126,17 +126,51 @@ index dc37ac4a018a086c4244a09a67215dbaa9b4de65..fc80522666f91087ce1bce3a34844b17 style: { display: "flex", flexDirection: "column", -@@ -971,9 +1002,10 @@ var SelectItemText = React.forwardRef( +@@ -864,10 +895,15 @@ var SelectItem = React.forwardRef( + const contentContext = useSelectContentContext(ITEM_NAME, __scopeSelect); + const isSelected = context.value === value; + const [textValue, setTextValue] = React.useState(textValueProp ?? ""); ++ const textValueRef = React.useRef(textValueProp ?? ""); + const [isFocused, setIsFocused] = React.useState(false); ++ const itemRefCallback = React.useCallback( ++ (node) => contentContext.itemRefCallback?.(node, value, disabled), ++ [contentContext.itemRefCallback, value, disabled] ++ ); + const composedRefs = (0, import_react_compose_refs.useComposedRefs)( + forwardedRef, +- (node) => contentContext.itemRefCallback?.(node, value, disabled) ++ itemRefCallback + ); + const textId = (0, import_react_id.useId)(); + const pointerTypeRef = React.useRef("touch"); +@@ -893,7 +931,10 @@ var SelectItem = React.forwardRef( + textId, + isSelected, + onItemTextChange: React.useCallback((node) => { +- setTextValue((prevTextValue) => prevTextValue || (node?.textContent ?? "").trim()); ++ const nextTextValue = (node?.textContent ?? "").trim(); ++ if (!nextTextValue || textValueRef.current) return; ++ textValueRef.current = nextTextValue; ++ setTextValue(nextTextValue); + }, []), + children: /* @__PURE__ */ (0, import_jsx_runtime.jsx)( + Collection.ItemSlot, +@@ -971,9 +1013,14 @@ var SelectItemText = React.forwardRef( const itemContext = useSelectItemContext(ITEM_TEXT_NAME, __scopeSelect); const nativeOptionsContext = useSelectNativeOptionsContext(ITEM_TEXT_NAME, __scopeSelect); const [itemTextNode, setItemTextNode] = React.useState(null); + const setItemTextNodeRef = useGuardedNodeSetter(setItemTextNode); ++ const itemTextRefCallback = React.useCallback( ++ (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled), ++ [contentContext.itemTextRefCallback, itemContext.value, itemContext.disabled] ++ ); const composedRefs = (0, import_react_compose_refs.useComposedRefs)( forwardedRef, - (node) => setItemTextNode(node), + setItemTextNodeRef, itemContext.onItemTextChange, - (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled) +- (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled) ++ itemTextRefCallback ); diff --git a/dist/index.mjs b/dist/index.mjs index f9b94f39dfddef678ef3086354f8d7413ae27e52..4a53ec0d65aa051b95e3e8ea4aff4fdd52a17406 100644 @@ -266,15 +300,49 @@ index f9b94f39dfddef678ef3086354f8d7413ae27e52..4a53ec0d65aa051b95e3e8ea4aff4fdd style: { display: "flex", flexDirection: "column", -@@ -904,9 +935,10 @@ var SelectItemText = React.forwardRef( +@@ -797,10 +828,15 @@ var SelectItem = React.forwardRef( + const contentContext = useSelectContentContext(ITEM_NAME, __scopeSelect); + const isSelected = context.value === value; + const [textValue, setTextValue] = React.useState(textValueProp ?? ""); ++ const textValueRef = React.useRef(textValueProp ?? ""); + const [isFocused, setIsFocused] = React.useState(false); ++ const itemRefCallback = React.useCallback( ++ (node) => contentContext.itemRefCallback?.(node, value, disabled), ++ [contentContext.itemRefCallback, value, disabled] ++ ); + const composedRefs = useComposedRefs( + forwardedRef, +- (node) => contentContext.itemRefCallback?.(node, value, disabled) ++ itemRefCallback + ); + const textId = useId(); + const pointerTypeRef = React.useRef("touch"); +@@ -826,7 +864,10 @@ var SelectItem = React.forwardRef( + textId, + isSelected, + onItemTextChange: React.useCallback((node) => { +- setTextValue((prevTextValue) => prevTextValue || (node?.textContent ?? "").trim()); ++ const nextTextValue = (node?.textContent ?? "").trim(); ++ if (!nextTextValue || textValueRef.current) return; ++ textValueRef.current = nextTextValue; ++ setTextValue(nextTextValue); + }, []), + children: /* @__PURE__ */ jsx( + Collection.ItemSlot, +@@ -904,9 +946,14 @@ var SelectItemText = React.forwardRef( const itemContext = useSelectItemContext(ITEM_TEXT_NAME, __scopeSelect); const nativeOptionsContext = useSelectNativeOptionsContext(ITEM_TEXT_NAME, __scopeSelect); const [itemTextNode, setItemTextNode] = React.useState(null); + const setItemTextNodeRef = useGuardedNodeSetter(setItemTextNode); ++ const itemTextRefCallback = React.useCallback( ++ (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled), ++ [contentContext.itemTextRefCallback, itemContext.value, itemContext.disabled] ++ ); const composedRefs = useComposedRefs( forwardedRef, - (node) => setItemTextNode(node), + setItemTextNodeRef, itemContext.onItemTextChange, - (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled) +- (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled) ++ itemTextRefCallback ); diff --git a/patches/@radix-ui__react-slot@1.2.3.patch b/patches/@radix-ui__react-slot@1.2.3.patch new file mode 100644 index 00000000..16998b69 --- /dev/null +++ b/patches/@radix-ui__react-slot@1.2.3.patch @@ -0,0 +1,44 @@ +diff --git a/dist/index.js b/dist/index.js +index 6a29e17b2246048ddb3da22732e2c81517bf81da..cecc799949dbbe10ba98d587fdad5d3a9e1d8cf1 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -70,11 +70,15 @@ var Slot = /* @__PURE__ */ createSlot("Slot"); + function createSlotClone(ownerName) { + const SlotClone = React.forwardRef((props, forwardedRef) => { + const { children, ...slotProps } = props; ++ const childrenRef = React.isValidElement(children) ? getElementRef(children) : null; ++ const composedRef = React.useMemo( ++ () => forwardedRef && childrenRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : forwardedRef || childrenRef || null, ++ [forwardedRef, childrenRef] ++ ); + if (React.isValidElement(children)) { +- const childrenRef = getElementRef(children); + const props2 = mergeProps(slotProps, children.props); + if (children.type !== React.Fragment) { +- props2.ref = forwardedRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : childrenRef; ++ props2.ref = composedRef; + } + return React.cloneElement(children, props2); + } +diff --git a/dist/index.mjs b/dist/index.mjs +index 0d5bcb5df0a670f429c157c5354c7b0499e9a599..4b4cbbf55bdfb4ee3f2a91aa02ff1ffef51d98cf 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -30,11 +30,15 @@ var Slot = /* @__PURE__ */ createSlot("Slot"); + function createSlotClone(ownerName) { + const SlotClone = React.forwardRef((props, forwardedRef) => { + const { children, ...slotProps } = props; ++ const childrenRef = React.isValidElement(children) ? getElementRef(children) : null; ++ const composedRef = React.useMemo( ++ () => forwardedRef && childrenRef ? composeRefs(forwardedRef, childrenRef) : forwardedRef || childrenRef || null, ++ [forwardedRef, childrenRef] ++ ); + if (React.isValidElement(children)) { +- const childrenRef = getElementRef(children); + const props2 = mergeProps(slotProps, children.props); + if (children.type !== React.Fragment) { +- props2.ref = forwardedRef ? composeRefs(forwardedRef, childrenRef) : childrenRef; ++ props2.ref = composedRef; + } + return React.cloneElement(children, props2); + } diff --git a/patches/@radix-ui__react-slot@1.2.4.patch b/patches/@radix-ui__react-slot@1.2.4.patch new file mode 100644 index 00000000..c274a099 --- /dev/null +++ b/patches/@radix-ui__react-slot@1.2.4.patch @@ -0,0 +1,48 @@ +diff --git a/dist/index.js b/dist/index.js +index 997ad803d345479c6afedb38fbfa4fed36dbb69f..e3845eab81088ac07f86ce0758f8fbb51bf20f7e 100644 +--- a/dist/index.js ++++ b/dist/index.js +@@ -87,13 +87,17 @@ function createSlotClone(ownerName) { + if (isLazyComponent(children) && typeof use === "function") { + children = use(children._payload); + } ++ const childrenRef = React.isValidElement(children) ? getElementRef(children) : null; ++ const composedRef = React.useMemo( ++ () => forwardedRef && childrenRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : forwardedRef || childrenRef || null, ++ [forwardedRef, childrenRef] ++ ); + if (React.isValidElement(children)) { +- const childrenRef = getElementRef(children); + const props2 = mergeProps(slotProps, children.props); + if (children.type !== React.Fragment) { +- props2.ref = forwardedRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : childrenRef; ++ props2.ref = composedRef; + } + return React.cloneElement(children, props2); + } + return React.Children.count(children) > 1 ? React.Children.only(null) : null; + }); +diff --git a/dist/index.mjs b/dist/index.mjs +index 6365c8003f76c8a41351ddc65a4e84ad8b6bf0d1..3e6146372c27ae6dc1a628264e49910b9f41f3f7 100644 +--- a/dist/index.mjs ++++ b/dist/index.mjs +@@ -46,13 +46,17 @@ function createSlotClone(ownerName) { + if (isLazyComponent(children) && typeof use === "function") { + children = use(children._payload); + } ++ const childrenRef = React.isValidElement(children) ? getElementRef(children) : null; ++ const composedRef = React.useMemo( ++ () => forwardedRef && childrenRef ? composeRefs(forwardedRef, childrenRef) : forwardedRef || childrenRef || null, ++ [forwardedRef, childrenRef] ++ ); + if (React.isValidElement(children)) { +- const childrenRef = getElementRef(children); + const props2 = mergeProps(slotProps, children.props); + if (children.type !== React.Fragment) { +- props2.ref = forwardedRef ? composeRefs(forwardedRef, childrenRef) : childrenRef; ++ props2.ref = composedRef; + } + return React.cloneElement(children, props2); + } + return React.Children.count(children) > 1 ? React.Children.only(null) : null; + }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 427707d1..96bfbe35 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -68,8 +68,14 @@ patchedDependencies: hash: afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e path: patches/@radix-ui__react-presence@1.1.5.patch '@radix-ui/react-select@2.2.6': - hash: eea50c1407feb65af64720d0aadd2b534ca7743474d9204e0fc1d6b93e5f31f4 + hash: 93cdd02c858fd8e3669eb902abbbd15644c2581c4c33aa63862fa351fefb4231 path: patches/@radix-ui__react-select@2.2.6.patch + '@radix-ui/react-slot@1.2.3': + hash: cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504 + path: patches/@radix-ui__react-slot@1.2.3.patch + '@radix-ui/react-slot@1.2.4': + hash: 5c525e90054caa3bbfa5599b10f7650914e2085f54b773bd115ad0e41eeca30a + path: patches/@radix-ui__react-slot@1.2.4.patch '@radix-ui/react-tooltip@1.2.8': hash: 92cb648a695f616d3b7222b90053cb36e162bab4303abf0fe39b517e1d9dd6b8 path: patches/@radix-ui__react-tooltip@1.2.8.patch @@ -209,10 +215,10 @@ importers: version: 1.1.5(patch_hash=afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-select': specifier: ^2.2.6 - version: 2.2.6(patch_hash=eea50c1407feb65af64720d0aadd2b534ca7743474d9204e0fc1d6b93e5f31f4)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + version: 2.2.6(patch_hash=93cdd02c858fd8e3669eb902abbbd15644c2581c4c33aa63862fa351fefb4231)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-slot': specifier: ^1.2.4 - version: 1.2.4(@types/react@19.2.14)(react@19.2.4) + version: 1.2.4(patch_hash=5c525e90054caa3bbfa5599b10f7650914e2085f54b773bd115ad0e41eeca30a)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-tabs': specifier: ^1.1.13 version: 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -14462,7 +14468,7 @@ snapshots: '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-dialog': 1.1.15(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) optionalDependencies: @@ -14515,7 +14521,7 @@ snapshots: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-context': 1.1.2(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) optionalDependencies: @@ -14560,7 +14566,7 @@ snapshots: '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-presence': 1.1.5(patch_hash=afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.4) aria-hidden: 1.2.6 react: 19.2.4 @@ -14670,7 +14676,7 @@ snapshots: '@radix-ui/react-presence': 1.1.5(patch_hash=afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-roving-focus': 1.1.11(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.4) aria-hidden: 1.2.6 react: 19.2.4 @@ -14693,7 +14699,7 @@ snapshots: '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-presence': 1.1.5(patch_hash=afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.4) aria-hidden: 1.2.6 react: 19.2.4 @@ -14743,7 +14749,7 @@ snapshots: '@radix-ui/react-primitive@2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) optionalDependencies: @@ -14752,7 +14758,7 @@ snapshots: '@radix-ui/react-primitive@2.1.4(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: - '@radix-ui/react-slot': 1.2.4(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.4(patch_hash=5c525e90054caa3bbfa5599b10f7650914e2085f54b773bd115ad0e41eeca30a)(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 react-dom: 19.2.4(react@19.2.4) optionalDependencies: @@ -14776,7 +14782,7 @@ snapshots: '@types/react': 19.2.14 '@types/react-dom': 19.2.3(@types/react@19.2.14) - '@radix-ui/react-select@2.2.6(patch_hash=eea50c1407feb65af64720d0aadd2b534ca7743474d9204e0fc1d6b93e5f31f4)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + '@radix-ui/react-select@2.2.6(patch_hash=93cdd02c858fd8e3669eb902abbbd15644c2581c4c33aa63862fa351fefb4231)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': dependencies: '@radix-ui/number': 1.1.1 '@radix-ui/primitive': 1.1.3 @@ -14791,7 +14797,7 @@ snapshots: '@radix-ui/react-popper': 1.2.8(patch_hash=bab709aa37cbdb3023036cd85534ddb1400f2fccfe54bd6321fe405d5d199404)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-callback-ref': 1.1.1(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-layout-effect': 1.1.1(@types/react@19.2.14)(react@19.2.4) @@ -14805,14 +14811,14 @@ snapshots: '@types/react': 19.2.14 '@types/react-dom': 19.2.3(@types/react@19.2.14) - '@radix-ui/react-slot@1.2.3(@types/react@19.2.14)(react@19.2.4)': + '@radix-ui/react-slot@1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4)': dependencies: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 optionalDependencies: '@types/react': 19.2.14 - '@radix-ui/react-slot@1.2.4(@types/react@19.2.14)(react@19.2.4)': + '@radix-ui/react-slot@1.2.4(patch_hash=5c525e90054caa3bbfa5599b10f7650914e2085f54b773bd115ad0e41eeca30a)(@types/react@19.2.14)(react@19.2.4)': dependencies: '@radix-ui/react-compose-refs': 1.1.2(@types/react@19.2.14)(react@19.2.4) react: 19.2.4 @@ -14846,7 +14852,7 @@ snapshots: '@radix-ui/react-portal': 1.1.9(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-presence': 1.1.5(patch_hash=afe90f800cfb3b1ce1a9c457772e2441a9202e1aa3f8658eb3b9613b3ba0ef7e)(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) '@radix-ui/react-primitive': 2.1.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) - '@radix-ui/react-slot': 1.2.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-slot': 1.2.3(patch_hash=cc153761e59f07565f64cf0883bb098c78bb27bed60b7363eaadb74ad4317504)(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-use-controllable-state': 1.2.2(@types/react@19.2.14)(react@19.2.4) '@radix-ui/react-visually-hidden': 1.2.3(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) react: 19.2.4 diff --git a/scripts/ci/enforce-pnpm-install.mjs b/scripts/ci/enforce-pnpm-install.mjs new file mode 100644 index 00000000..fb33357b --- /dev/null +++ b/scripts/ci/enforce-pnpm-install.mjs @@ -0,0 +1,13 @@ +const userAgent = process.env.npm_config_user_agent ?? ''; + +if (userAgent.startsWith('pnpm/')) { + process.exit(0); +} + +console.error( + [ + 'Use pnpm install for this project.', + 'npm and yarn do not apply pnpm patchedDependencies, including the Radix React 19 patches.', + ].join('\n') +); +process.exit(1); diff --git a/scripts/ci/verify-radix-presence-patch.mjs b/scripts/ci/verify-radix-presence-patch.mjs index e62b7515..0c5aec62 100644 --- a/scripts/ci/verify-radix-presence-patch.mjs +++ b/scripts/ci/verify-radix-presence-patch.mjs @@ -25,10 +25,19 @@ const patchChecks = [ }, { packageName: '@radix-ui/react-select', - requiredMarkers: ['useGuardedNodeSetter', 'setContentRef', 'setItemTextNodeRef'], + requiredMarkers: [ + 'useGuardedNodeSetter', + 'setContentRef', + 'setItemTextNodeRef', + 'textValueRef', + 'nextTextValue', + ], forbiddenSnippets: [ '(node) => setContent(node)', '(node) => setItemTextNode(node)', + 'forwardedRef,\n (node) => contentContext.itemRefCallback?.(node, value, disabled)', + 'itemContext.onItemTextChange,\n (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled)', + 'setTextValue((prevTextValue) => prevTextValue || (node?.textContent ?? "").trim());', 'onTriggerChange: setTrigger,', 'onValueNodeChange: setValueNode,', 'onViewportChange: setViewport,', @@ -37,6 +46,23 @@ const patchChecks = [ 'setSelectedItemText(node);', ], }, + { + packageName: '@radix-ui/react-slot', + requiredMarkers: ['composedRef', 'React.useMemo'], + forbiddenSnippets: [ + 'props2.ref = forwardedRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : childrenRef;', + 'props2.ref = forwardedRef ? composeRefs(forwardedRef, childrenRef) : childrenRef;', + ], + }, + { + packageName: '@radix-ui/react-slot', + resolverFromPackage: '@radix-ui/react-select', + requiredMarkers: ['composedRef', 'React.useMemo'], + forbiddenSnippets: [ + 'props2.ref = forwardedRef ? (0, import_react_compose_refs.composeRefs)(forwardedRef, childrenRef) : childrenRef;', + 'props2.ref = forwardedRef ? composeRefs(forwardedRef, childrenRef) : childrenRef;', + ], + }, { packageName: '@radix-ui/react-popper', resolverFromPackage: '@radix-ui/react-select', diff --git a/scripts/ci/verify-radix-renderer-bundle.mjs b/scripts/ci/verify-radix-renderer-bundle.mjs index 527c4763..528146d1 100644 --- a/scripts/ci/verify-radix-renderer-bundle.mjs +++ b/scripts/ci/verify-radix-renderer-bundle.mjs @@ -27,6 +27,9 @@ const requiredMarkers = [ 'setSelectedItemRef', 'setSelectedItemTextRef', 'setItemTextNodeRef', + 'textValueRef', + 'nextTextValue', + 'composedRef', 'setControlRef', 'setBubbleInputRef', ]; @@ -35,6 +38,12 @@ const forbiddenSnippets = [ '(node) => setContent(node)', '(node2) => setNode(node2)', '(node) => setItemTextNode(node)', + 'forwardedRef,\n (node) => contentContext.itemRefCallback?.(node, value, disabled)', + 'forwardedRef,\n (node2) => contentContext.itemRefCallback?.(node2, value, disabled)', + 'itemContext.onItemTextChange,\n (node) => contentContext.itemTextRefCallback?.(node, itemContext.value, itemContext.disabled)', + 'itemContext.onItemTextChange,\n (node2) => contentContext.itemTextRefCallback?.(node2, itemContext.value, itemContext.disabled)', + 'setTextValue((prevTextValue) => prevTextValue || (node?.textContent ?? "").trim());', + 'setTextValue((prevTextValue) => prevTextValue || (node2?.textContent ?? "").trim());', 'onContentChange: setContent,', 'onTriggerChange: setTrigger,', 'onValueNodeChange: setValueNode,', @@ -46,6 +55,7 @@ const forbiddenSnippets = [ 'useComposedRefs(forwardedRef, setBubbleInput)', 'useComposedRefs)(forwardedRef, setControl)', 'useComposedRefs)(forwardedRef, setBubbleInput)', + 'props2.ref = forwardedRef ? composeRefs(forwardedRef, childrenRef) : childrenRef;', ]; const failures = []; diff --git a/src/features/codex-account/renderer/mergeCodexProviderStatusWithSnapshot.ts b/src/features/codex-account/renderer/mergeCodexProviderStatusWithSnapshot.ts index 620cd1f7..25a65fe7 100644 --- a/src/features/codex-account/renderer/mergeCodexProviderStatusWithSnapshot.ts +++ b/src/features/codex-account/renderer/mergeCodexProviderStatusWithSnapshot.ts @@ -168,6 +168,11 @@ export function mergeCodexProviderStatusWithSnapshot( } const availableBackends = mergeCodexNativeBackendOption(provider, snapshot); + const customProvider = provider.connection?.codex?.customProvider ?? null; + const endpointLabel = + customProvider?.active === true && customProvider.baseUrl.trim() + ? customProvider.baseUrl.trim() + : 'codex exec --json'; const baseConnection = provider.connection ?? { supportsOAuth: false, supportsApiKey: true, @@ -203,7 +208,7 @@ export function mergeCodexProviderStatusWithSnapshot( backend: { kind: CODEX_NATIVE_BACKEND_ID, label: CODEX_NATIVE_LABEL, - endpointLabel: 'codex exec --json', + endpointLabel, projectId: provider.backend?.projectId ?? null, authMethodDetail: snapshot.effectiveAuthMode ?? null, }, @@ -227,6 +232,13 @@ export function mergeCodexProviderStatusWithSnapshot( localActiveChatgptAccountPresent: snapshot.localActiveChatgptAccountPresent, login: snapshot.login, rateLimits: snapshot.rateLimits, + customProvider: customProvider ?? { + enabled: false, + active: false, + baseUrl: '', + model: '', + issueMessage: null, + }, }, }, }; diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncDiagnosticsReader.ts b/src/features/member-work-sync/core/application/MemberWorkSyncDiagnosticsReader.ts index aaa4915d..c329abe1 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncDiagnosticsReader.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncDiagnosticsReader.ts @@ -1,6 +1,7 @@ import { decideMemberWorkSyncStatus } from '../domain'; import { finalizeMemberWorkSyncAgenda } from './MemberWorkSyncReconciler'; +import { resolveMemberWorkSyncRuntimeActivity } from './MemberWorkSyncRuntimeActivity'; import type { MemberWorkSyncStatus, MemberWorkSyncStatusRequest } from '../../contracts'; import type { MemberWorkSyncUseCaseDeps } from './ports'; @@ -17,13 +18,14 @@ export class MemberWorkSyncDiagnosticsReader { const source = await this.deps.agendaSource.loadAgenda(request); const agenda = finalizeMemberWorkSyncAgenda(this.deps, source); const nowIso = this.deps.clock.now().toISOString(); - const teamActive = this.deps.lifecycle - ? await this.deps.lifecycle.isTeamActive(agenda.teamName) - : true; + const runtimeActivity = await resolveMemberWorkSyncRuntimeActivity(this.deps, { + teamName: agenda.teamName, + memberName: agenda.memberName, + }); const decision = decideMemberWorkSyncStatus({ agenda, nowIso, - inactive: source.inactive || !teamActive, + inactive: source.inactive || runtimeActivity.inactive, }); return { @@ -39,7 +41,7 @@ export class MemberWorkSyncDiagnosticsReader { evaluatedAt: nowIso, diagnostics: [ ...agenda.diagnostics, - ...(!teamActive ? ['team_runtime_inactive'] : []), + ...runtimeActivity.diagnostics, ...decision.diagnostics, 'status_snapshot_not_persisted', ], diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts index dcbc4f3e..038ae15e 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeDispatcher.ts @@ -3,6 +3,7 @@ import { decideMemberWorkSyncStatus } from '../domain'; import { appendMemberWorkSyncAudit, reasonToAuditEvent } from './MemberWorkSyncAudit'; import { decideMemberWorkSyncNudgeActivation } from './MemberWorkSyncNudgeActivationPolicy'; import { finalizeMemberWorkSyncAgenda } from './MemberWorkSyncReconciler'; +import { resolveMemberWorkSyncRuntimeActivity } from './MemberWorkSyncRuntimeActivity'; import type { MemberWorkSyncAgenda, @@ -14,6 +15,9 @@ import type { MemberWorkSyncAuditEventName, MemberWorkSyncUseCaseDeps } from './ const MEMBER_WORK_SYNC_MAX_NUDGES_PER_MEMBER_PER_HOUR = 2; const MEMBER_WORK_SYNC_RETRY_BASE_MINUTES = 10; const MEMBER_WORK_SYNC_RETRY_MAX_MINUTES = 60; +const MEMBER_WORK_SYNC_NUDGE_DISPATCH_ITEM_TIMEOUT_MS = 2 * 60_000; +const MEMBER_WORK_SYNC_NUDGE_DISPATCH_TEAM_TIMEOUT_MS = 2 * 60_000; +const MEMBER_WORK_SYNC_NUDGE_CLAIM_TIMEOUT_MS = 30_000; const AGENDA_SYNC_STILL_STUCK_RECOVERY_INTENT_PREFIX = 'agenda-sync-still-stuck:'; export interface MemberWorkSyncNudgeDispatchSummary { @@ -28,12 +32,32 @@ export interface MemberWorkSyncNudgeDispatchOptions { claimedBy: string; teamNames: string[]; limit?: number; + itemTimeoutMs?: number; + teamTimeoutMs?: number; + claimTimeoutMs?: number; } function emptySummary(): MemberWorkSyncNudgeDispatchSummary { return { claimed: 0, delivered: 0, superseded: 0, retryable: 0, terminal: 0 }; } +function addSummary( + left: MemberWorkSyncNudgeDispatchSummary, + right: MemberWorkSyncNudgeDispatchSummary +): MemberWorkSyncNudgeDispatchSummary { + return { + claimed: left.claimed + right.claimed, + delivered: left.delivered + right.delivered, + superseded: left.superseded + right.superseded, + retryable: left.retryable + right.retryable, + terminal: left.terminal + right.terminal, + }; +} + +function unrefTimer(timer: ReturnType): void { + timer.unref?.(); +} + function addMinutes(iso: string, minutes: number): string { return new Date(Date.parse(iso) + minutes * 60_000).toISOString(); } @@ -116,6 +140,22 @@ function reviewPickupRequestIdsStillMatch( return payloadIds.length > 0 && payloadIds.every((id) => agendaIds.includes(id)); } +interface MemberWorkSyncNudgeDispatchRun { + cancelled: boolean; + parent?: MemberWorkSyncNudgeDispatchRun; +} + +function isDispatchRunCancelled(run?: MemberWorkSyncNudgeDispatchRun): boolean { + let current: MemberWorkSyncNudgeDispatchRun | undefined = run; + while (current) { + if (current.cancelled) { + return true; + } + current = current.parent; + } + return false; +} + export class MemberWorkSyncNudgeDispatcher { constructor(private readonly deps: MemberWorkSyncUseCaseDeps) {} @@ -129,28 +169,275 @@ export class MemberWorkSyncNudgeDispatcher { } const nowIso = this.deps.clock.now().toISOString(); - const summary = emptySummary(); - for (const teamName of [ - ...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean)), - ]) { - const claimed = await outbox.claimDue({ - teamName, - claimedBy: options.claimedBy, - nowIso, - limit: options.limit ?? 10, - }); - summary.claimed += claimed.length; - for (const item of claimed) { - const result = await this.dispatchItem(item, nowIso); - summary[result] += 1; + const itemTimeoutMs = Math.max( + 1, + options.itemTimeoutMs ?? MEMBER_WORK_SYNC_NUDGE_DISPATCH_ITEM_TIMEOUT_MS + ); + const teamTimeoutMs = Math.max( + 1, + options.teamTimeoutMs ?? MEMBER_WORK_SYNC_NUDGE_DISPATCH_TEAM_TIMEOUT_MS + ); + const claimTimeoutMs = Math.max( + 1, + options.claimTimeoutMs ?? MEMBER_WORK_SYNC_NUDGE_CLAIM_TIMEOUT_MS + ); + const teamNames = [...new Set(options.teamNames.map((name) => name.trim()).filter(Boolean))]; + const summaries = await Promise.allSettled( + teamNames.map((teamName) => + this.dispatchTeamWithTimeout(teamName, options, nowIso, { + itemTimeoutMs, + teamTimeoutMs, + claimTimeoutMs, + }) + ) + ); + + let summary = emptySummary(); + for (const [index, result] of summaries.entries()) { + if (result.status === 'fulfilled') { + summary = addSummary(summary, result.value); + } else { + this.deps.logger?.warn('member work sync team nudge dispatch failed', { + teamName: teamNames[index], + error: String(result.reason), + }); } } return summary; } + private async dispatchTeamWithTimeout( + teamName: string, + options: MemberWorkSyncNudgeDispatchOptions, + nowIso: string, + timeouts: { itemTimeoutMs: number; teamTimeoutMs: number; claimTimeoutMs: number } + ): Promise { + let timeout: ReturnType | null = null; + const run: MemberWorkSyncNudgeDispatchRun = { cancelled: false }; + const work = this.dispatchTeam(teamName, options, nowIso, timeouts, run); + void work.catch(() => undefined); + + try { + const result = await Promise.race([ + work, + new Promise<'timeout'>((resolve) => { + timeout = setTimeout(() => { + run.cancelled = true; + resolve('timeout'); + }, timeouts.teamTimeoutMs); + unrefTimer(timeout); + }), + ]); + if (result !== 'timeout') { + return result; + } + this.deps.logger?.warn('member work sync team nudge dispatch timed out', { + teamName, + timeoutMs: timeouts.teamTimeoutMs, + }); + return emptySummary(); + } finally { + run.cancelled = true; + if (timeout) { + clearTimeout(timeout); + } + } + } + + private async dispatchTeam( + teamName: string, + options: MemberWorkSyncNudgeDispatchOptions, + nowIso: string, + timeouts: { itemTimeoutMs: number; claimTimeoutMs: number }, + run: MemberWorkSyncNudgeDispatchRun + ): Promise { + const summary = emptySummary(); + const claimed = await this.claimDueWithTimeout(teamName, options, nowIso, timeouts, run); + if (!claimed || isDispatchRunCancelled(run)) { + return summary; + } + + summary.claimed += claimed.length; + for (const item of claimed) { + if (isDispatchRunCancelled(run)) { + break; + } + const result = await this.dispatchItemWithTimeout(item, nowIso, timeouts.itemTimeoutMs, run); + summary[result] += 1; + } + return summary; + } + + private async claimDueWithTimeout( + teamName: string, + options: MemberWorkSyncNudgeDispatchOptions, + nowIso: string, + timeouts: { claimTimeoutMs: number }, + run: MemberWorkSyncNudgeDispatchRun + ): Promise { + const outbox = this.deps.outboxStore; + if (!outbox) { + return null; + } + + let timeout: ReturnType | null = null; + const work = outbox.claimDue({ + teamName, + claimedBy: options.claimedBy, + nowIso, + limit: options.limit ?? 10, + }); + void work.catch(() => undefined); + + try { + const result = await Promise.race([ + work, + new Promise<'timeout'>((resolve) => { + timeout = setTimeout(() => resolve('timeout'), timeouts.claimTimeoutMs); + unrefTimer(timeout); + }), + ]); + if (result !== 'timeout') { + return isDispatchRunCancelled(run) ? null : result; + } + this.deps.logger?.warn('member work sync nudge claim timed out', { + teamName, + timeoutMs: timeouts.claimTimeoutMs, + }); + return null; + } catch (error) { + this.deps.logger?.warn('member work sync nudge claim failed', { + teamName, + error: String(error), + }); + return null; + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } + + private async dispatchItemWithTimeout( + item: MemberWorkSyncOutboxItem, + nowIso: string, + timeoutMs: number, + run: MemberWorkSyncNudgeDispatchRun + ): Promise> { + let timeout: ReturnType | null = null; + const itemRun: MemberWorkSyncNudgeDispatchRun = { cancelled: false, parent: run }; + const work = this.dispatchItem(item, nowIso, itemRun); + void work.catch(() => undefined); + + try { + const result = await Promise.race< + keyof Omit | 'timeout' + >([ + work, + new Promise<'timeout'>((resolve) => { + timeout = setTimeout(() => { + itemRun.cancelled = true; + resolve('timeout'); + }, timeoutMs); + unrefTimer(timeout); + }), + ]); + if (result !== 'timeout') { + return result; + } + await this.tryMarkDispatchItemRetryable( + item, + nowIso, + `nudge dispatch item timed out after ${timeoutMs}ms`, + timeoutMs, + run + ); + return 'retryable'; + } catch (error) { + await this.tryMarkDispatchItemRetryable(item, nowIso, String(error), timeoutMs, run); + return 'retryable'; + } finally { + itemRun.cancelled = true; + if (timeout) { + clearTimeout(timeout); + } + } + } + + private async tryMarkDispatchItemRetryable( + item: MemberWorkSyncOutboxItem, + nowIso: string, + error: string, + timeoutMs: number, + run?: MemberWorkSyncNudgeDispatchRun + ): Promise { + if (isDispatchRunCancelled(run)) { + return; + } + let timeout: ReturnType | null = null; + const markTimeoutMs = Math.min(Math.max(1, timeoutMs), 5_000); + const work = this.markDispatchItemRetryable(item, nowIso, error, run); + void work.catch(() => undefined); + + try { + const result = await Promise.race([ + work.then(() => 'marked' as const), + new Promise<'timeout'>((resolve) => { + timeout = setTimeout(() => resolve('timeout'), markTimeoutMs); + unrefTimer(timeout); + }), + ]); + if (result === 'timeout') { + this.deps.logger?.warn('member work sync nudge retry mark timed out', { + teamName: item.teamName, + memberName: item.memberName, + outboxId: item.id, + timeoutMs: markTimeoutMs, + error, + }); + } + } catch (markError) { + this.deps.logger?.warn('member work sync nudge retry mark failed', { + teamName: item.teamName, + memberName: item.memberName, + outboxId: item.id, + error: String(markError), + }); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } + + private async markDispatchItemRetryable( + item: MemberWorkSyncOutboxItem, + nowIso: string, + error: string, + run?: MemberWorkSyncNudgeDispatchRun + ): Promise { + if (isDispatchRunCancelled(run)) { + return; + } + await this.deps.outboxStore?.markFailed({ + teamName: item.teamName, + id: item.id, + attemptGeneration: item.attemptGeneration, + error, + retryable: true, + nowIso, + nextAttemptAt: nextRetryAt(item, nowIso), + }); + if (isDispatchRunCancelled(run)) { + return; + } + await this.appendDispatchAudit(item, 'nudge_retryable', error); + } + private async dispatchItem( item: MemberWorkSyncOutboxItem, - nowIso: string + nowIso: string, + run: MemberWorkSyncNudgeDispatchRun ): Promise> { const outbox = this.deps.outboxStore; const inbox = this.deps.inboxNudge; @@ -158,7 +445,13 @@ export class MemberWorkSyncNudgeDispatcher { return 'terminal'; } + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } const revalidation = await this.revalidate(item, nowIso); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } if (!revalidation.ok) { if (revalidation.retryable) { await outbox.markFailed({ @@ -170,6 +463,9 @@ export class MemberWorkSyncNudgeDispatcher { nowIso, nextAttemptAt: revalidation.nextAttemptAt ?? nextRetryAt(item, nowIso), }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit( item, reasonToAuditEvent(revalidation.reason), @@ -178,8 +474,8 @@ export class MemberWorkSyncNudgeDispatcher { return 'retryable'; } if (revalidation.reason.startsWith('review_pickup_delivery_unavailable:')) { - await this.markReviewPickupDeliveryUnavailable(item, nowIso, revalidation.reason); - return 'superseded'; + await this.markReviewPickupDeliveryUnavailable(item, nowIso, revalidation.reason, run); + return isDispatchRunCancelled(run) ? 'retryable' : 'superseded'; } await outbox.markSuperseded({ teamName: item.teamName, @@ -187,11 +483,17 @@ export class MemberWorkSyncNudgeDispatcher { reason: revalidation.reason, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_superseded', revalidation.reason); return 'superseded'; } try { + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } const inserted = await inbox.insertIfAbsent({ teamName: item.teamName, memberName: item.memberName, @@ -200,6 +502,9 @@ export class MemberWorkSyncNudgeDispatcher { payload: item.payload, timestamp: nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } if (inserted.conflict) { await outbox.markFailed({ teamName: item.teamName, @@ -209,6 +514,9 @@ export class MemberWorkSyncNudgeDispatcher { retryable: false, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_skipped', 'inbox_payload_conflict'); return 'terminal'; } @@ -218,7 +526,8 @@ export class MemberWorkSyncNudgeDispatcher { inserted.messageId, inserted.inserted, revalidation.providerId, - nowIso + nowIso, + run ); } await outbox.markDelivered({ @@ -228,15 +537,25 @@ export class MemberWorkSyncNudgeDispatcher { deliveredMessageId: inserted.messageId, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_delivered', 'inbox_inserted'); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.scheduleDeliveryWake( item, inserted.messageId, inserted.inserted, - revalidation.providerId + revalidation.providerId, + run ); - return 'delivered'; + return isDispatchRunCancelled(run) ? 'retryable' : 'delivered'; } catch (error) { + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await outbox.markFailed({ teamName: item.teamName, id: item.id, @@ -246,6 +565,9 @@ export class MemberWorkSyncNudgeDispatcher { nowIso, nextAttemptAt: nextRetryAt(item, nowIso), }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_retryable', String(error)); return 'retryable'; } @@ -256,7 +578,8 @@ export class MemberWorkSyncNudgeDispatcher { messageId: string, inserted: boolean, providerId: MemberWorkSyncStatus['providerId'] | undefined, - nowIso: string + nowIso: string, + run: MemberWorkSyncNudgeDispatchRun ): Promise> { const outbox = this.deps.outboxStore; const delivery = this.deps.reviewPickupDelivery; @@ -264,11 +587,15 @@ export class MemberWorkSyncNudgeDispatcher { await this.markReviewPickupDeliveryUnavailable( item, nowIso, - 'review_pickup_delivery_port_unavailable' + 'review_pickup_delivery_port_unavailable', + run ); - return 'superseded'; + return isDispatchRunCancelled(run) ? 'retryable' : 'superseded'; } + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } const outcome = await delivery.deliver({ teamName: item.teamName, memberName: item.memberName, @@ -278,6 +605,9 @@ export class MemberWorkSyncNudgeDispatcher { inserted, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } if (outcome.ok) { await outbox.markDelivered({ @@ -289,7 +619,13 @@ export class MemberWorkSyncNudgeDispatcher { deliveryDiagnostics: outcome.diagnostics, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'review_pickup_member_nudge_delivered', outcome.state); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_delivered', `review_pickup:${outcome.state}`); return 'delivered'; } @@ -304,13 +640,16 @@ export class MemberWorkSyncNudgeDispatcher { nowIso, nextAttemptAt: outcome.retryAfterIso ?? nextRetryAt(item, nowIso), }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'review_pickup_wake_failed_retryable', outcome.message); return 'retryable'; } if (outcome.reason === 'capability_absent') { - await this.markReviewPickupDeliveryUnavailable(item, nowIso, outcome.message); - return 'superseded'; + await this.markReviewPickupDeliveryUnavailable(item, nowIso, outcome.message, run); + return isDispatchRunCancelled(run) ? 'retryable' : 'superseded'; } await outbox.markFailed({ @@ -321,6 +660,9 @@ export class MemberWorkSyncNudgeDispatcher { retryable: false, nowIso, }); + if (isDispatchRunCancelled(run)) { + return 'retryable'; + } await this.appendDispatchAudit(item, 'nudge_skipped', outcome.message); return 'terminal'; } @@ -328,26 +670,40 @@ export class MemberWorkSyncNudgeDispatcher { private async markReviewPickupDeliveryUnavailable( item: MemberWorkSyncOutboxItem, nowIso: string, - reason: string + reason: string, + run?: MemberWorkSyncNudgeDispatchRun ): Promise { + if (isDispatchRunCancelled(run)) { + return; + } await this.deps.outboxStore?.markSuperseded({ teamName: item.teamName, id: item.id, reason, nowIso, }); + if (isDispatchRunCancelled(run)) { + return; + } await this.appendDispatchAudit(item, 'review_pickup_delivery_unavailable', reason); + if (isDispatchRunCancelled(run)) { + return; + } await this.appendDispatchAudit(item, 'review_pickup_escalated', reason); - await this.notifyReviewPickupEscalation(item, nowIso, reason); + if (isDispatchRunCancelled(run)) { + return; + } + await this.notifyReviewPickupEscalation(item, nowIso, reason, run); } private async notifyReviewPickupEscalation( item: MemberWorkSyncOutboxItem, nowIso: string, - reason: string + reason: string, + run?: MemberWorkSyncNudgeDispatchRun ): Promise { const escalation = this.deps.reviewPickupEscalation; - if (!escalation) { + if (!escalation || isDispatchRunCancelled(run)) { return; } @@ -395,12 +751,16 @@ export class MemberWorkSyncNudgeDispatcher { | { ok: true; providerId?: MemberWorkSyncStatus['providerId'] } | { ok: false; reason: string; retryable: boolean; nextAttemptAt?: string } > { - const teamActive = this.deps.lifecycle - ? await this.deps.lifecycle.isTeamActive(item.teamName) - : true; - if (!teamActive) { + const runtimeActivity = await resolveMemberWorkSyncRuntimeActivity(this.deps, { + teamName: item.teamName, + memberName: item.memberName, + }); + if (!runtimeActivity.teamActive) { return { ok: false, reason: 'team_inactive', retryable: false }; } + if (!runtimeActivity.memberActive) { + return { ok: false, reason: 'member_runtime_inactive', retryable: false }; + } const previous = await this.deps.statusStore.read({ teamName: item.teamName, @@ -424,7 +784,7 @@ export class MemberWorkSyncNudgeDispatcher { agenda, latestAcceptedReport: previous.report?.accepted ? previous.report : null, nowIso, - inactive: source.inactive || !teamActive, + inactive: source.inactive || runtimeActivity.inactive, }); const providerId = source.providerId ?? previous.providerId; const { report: _previousReport, ...previousWithoutReport } = previous; @@ -533,21 +893,46 @@ export class MemberWorkSyncNudgeDispatcher { } const taskIds = item.payload.taskRefs.map((taskRef) => taskRef.taskId); - if ( - this.deps.watchdogCooldown && - (await this.deps.watchdogCooldown.hasRecentNudge({ - teamName: item.teamName, - memberName: item.memberName, - taskIds, - nowIso, - })) - ) { - return { ok: false, reason: 'watchdog_cooldown_active', retryable: true }; + const watchdogCooldown = await this.resolveWatchdogCooldown(item, taskIds, nowIso); + if (watchdogCooldown.active) { + return { + ok: false, + reason: 'watchdog_cooldown_active', + retryable: true, + ...(watchdogCooldown.retryAfterIso + ? { nextAttemptAt: watchdogCooldown.retryAfterIso } + : {}), + }; } return { ok: true, ...(providerId ? { providerId } : {}) }; } + private async resolveWatchdogCooldown( + item: MemberWorkSyncOutboxItem, + taskIds: string[], + nowIso: string + ): Promise<{ active: boolean; retryAfterIso?: string }> { + const watchdogCooldown = this.deps.watchdogCooldown; + if (!watchdogCooldown) { + return { active: false }; + } + const input = { + teamName: item.teamName, + memberName: item.memberName, + taskIds, + nowIso, + }; + if (watchdogCooldown.getRecentNudgeCooldown) { + const result = await watchdogCooldown.getRecentNudgeCooldown(input); + return { + active: result.active, + ...(result.retryAfterIso ? { retryAfterIso: result.retryAfterIso } : {}), + }; + } + return { active: await watchdogCooldown.hasRecentNudge(input) }; + } + private async revalidateProofMissingRecovery( item: MemberWorkSyncOutboxItem, nowIso: string @@ -578,9 +963,10 @@ export class MemberWorkSyncNudgeDispatcher { item: MemberWorkSyncOutboxItem, messageId: string, inserted: boolean, - providerId?: MemberWorkSyncStatus['providerId'] + providerId?: MemberWorkSyncStatus['providerId'], + run?: MemberWorkSyncNudgeDispatchRun ): Promise { - if (!this.deps.nudgeDeliveryWake) { + if (!this.deps.nudgeDeliveryWake || isDispatchRunCancelled(run)) { return; } diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts index db6c3a8d..310c2f14 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncNudgeOutboxPlanner.ts @@ -56,6 +56,29 @@ function isTurnSettledReconcile(status: MemberWorkSyncStatus): boolean { return status.shadow?.triggerReasons?.includes('turn_settled') === true; } +function parseTime(value: string | undefined): number | null { + if (!value) { + return null; + } + const time = Date.parse(value); + return Number.isFinite(time) ? time : null; +} + +function hasActiveAcceptedWorkLease(status: MemberWorkSyncStatus): boolean { + const report = status.report; + if ( + report?.accepted !== true || + report.agendaFingerprint !== status.agenda.fingerprint || + (report.state !== 'still_working' && report.state !== 'blocked') + ) { + return false; + } + + const evaluatedAtMs = parseTime(status.evaluatedAt); + const expiresAtMs = parseTime(report.expiresAt); + return evaluatedAtMs != null && expiresAtMs != null && expiresAtMs > evaluatedAtMs; +} + function shouldPlanStatusOnlyRecovery(input: { status: MemberWorkSyncStatus; baseInput: MemberWorkSyncOutboxEnsureInput; @@ -68,7 +91,7 @@ function shouldPlanStatusOnlyRecovery(input: { input.baseInput.payload.workSyncIntent === 'agenda_sync' && input.baseInput.payload.workSyncIntentKey === undefined && input.existingItemStatus === 'delivered' && - input.status.report?.accepted !== true + !hasActiveAcceptedWorkLease(input.status) ); } @@ -84,18 +107,10 @@ function shouldPlanAgendaSyncRefreshRecovery(input: { input.baseInput.payload.workSyncIntentKey === undefined && input.existingItem.status === 'delivered' && input.existingItem.agendaFingerprint === input.baseInput.agendaFingerprint && - input.status.report?.accepted !== true + !hasActiveAcceptedWorkLease(input.status) ); } -function parseTime(value: string | undefined): number | null { - if (!value) { - return null; - } - const time = Date.parse(value); - return Number.isFinite(time) ? time : null; -} - function isDeliveredStillStuckRecoveryReason(reason: MemberWorkSyncNudgeActivationReason): boolean { return ( reason === 'shadow_ready' || @@ -125,7 +140,7 @@ function shouldPlanDeliveredStillStuckRecovery(input: { input.baseInput.payload.workSyncIntentKey !== undefined || !recoverableExistingItem || input.existingItem.agendaFingerprint !== input.baseInput.agendaFingerprint || - input.status.report?.accepted === true || + hasActiveAcceptedWorkLease(input.status) || !isDeliveredStillStuckRecoveryReason(input.activationReason) ) { return false; diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncPendingReportIntentReplayer.ts b/src/features/member-work-sync/core/application/MemberWorkSyncPendingReportIntentReplayer.ts index 8c6bce96..a878e463 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncPendingReportIntentReplayer.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncPendingReportIntentReplayer.ts @@ -17,7 +17,11 @@ function statusForResult(input: { if (input.accepted) { return 'accepted'; } - if (input.code === 'member_inactive' || input.code === 'team_runtime_inactive') { + if ( + input.code === 'member_inactive' || + input.code === 'team_runtime_inactive' || + input.code === 'member_runtime_inactive' + ) { return 'superseded'; } return 'rejected'; diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts b/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts index 06a33246..8c737920 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncReconciler.ts @@ -7,6 +7,7 @@ import { import { appendMemberWorkSyncAudit } from './MemberWorkSyncAudit'; import { MemberWorkSyncNudgeOutboxPlanner } from './MemberWorkSyncNudgeOutboxPlanner'; +import { resolveMemberWorkSyncRuntimeActivity } from './MemberWorkSyncRuntimeActivity'; import type { MemberWorkSyncStatus, MemberWorkSyncStatusRequest } from '../../contracts'; import type { MemberWorkSyncAgendaSourceResult, MemberWorkSyncUseCaseDeps } from './ports'; @@ -14,6 +15,7 @@ import type { MemberWorkSyncAgendaSourceResult, MemberWorkSyncUseCaseDeps } from export interface MemberWorkSyncReconcileContext { reconciledBy?: 'request' | 'queue'; triggerReasons?: string[]; + isCancelled?: () => boolean; recovery?: { kind: 'proof_missing'; intentKey: string; @@ -22,6 +24,19 @@ export interface MemberWorkSyncReconcileContext { }; } +export class MemberWorkSyncReconcileCancelledError extends Error { + constructor() { + super('member work sync reconcile cancelled'); + this.name = 'MemberWorkSyncReconcileCancelledError'; + } +} + +function assertReconcileNotCancelled(context: MemberWorkSyncReconcileContext): void { + if (context.isCancelled?.()) { + throw new MemberWorkSyncReconcileCancelledError(); + } +} + export function finalizeMemberWorkSyncAgenda( deps: MemberWorkSyncUseCaseDeps, source: MemberWorkSyncAgendaSourceResult @@ -61,6 +76,7 @@ export class MemberWorkSyncReconciler { ...(context.triggerReasons?.length ? { triggerReasons: context.triggerReasons } : {}), }); const source = await this.deps.agendaSource.loadAgenda(request); + assertReconcileNotCancelled(context); const agenda = finalizeMemberWorkSyncAgenda(this.deps, source); await appendMemberWorkSyncAudit(this.deps, { teamName: agenda.teamName, @@ -72,21 +88,24 @@ export class MemberWorkSyncReconciler { ...(source.providerId ? { providerId: source.providerId } : {}), diagnostics: agenda.diagnostics, }); + assertReconcileNotCancelled(context); const previous = await this.deps.statusStore.read(request); const nowIso = this.deps.clock.now().toISOString(); - const teamActive = this.deps.lifecycle - ? await this.deps.lifecycle.isTeamActive(agenda.teamName) - : true; + const runtimeActivity = await resolveMemberWorkSyncRuntimeActivity(this.deps, { + teamName: agenda.teamName, + memberName: agenda.memberName, + }); + assertReconcileNotCancelled(context); const decision = decideMemberWorkSyncStatus({ agenda, latestAcceptedReport: previous?.report?.accepted ? previous.report : null, nowIso, - inactive: source.inactive || !teamActive, + inactive: source.inactive || runtimeActivity.inactive, }); await appendMemberWorkSyncAudit(this.deps, { teamName: agenda.teamName, memberName: agenda.memberName, - event: source.inactive || !teamActive ? 'team_inactive' : 'decision_made', + event: source.inactive || runtimeActivity.inactive ? 'team_inactive' : 'decision_made', source: 'reconciler', agendaFingerprint: agenda.fingerprint, state: decision.state, @@ -95,6 +114,7 @@ export class MemberWorkSyncReconciler { diagnostics: decision.diagnostics, }); + assertReconcileNotCancelled(context); const status = await attachMemberWorkSyncReportToken(this.deps, { teamName: agenda.teamName, memberName: agenda.memberName, @@ -125,15 +145,13 @@ export class MemberWorkSyncReconciler { : {}), }, evaluatedAt: nowIso, - diagnostics: [ - ...agenda.diagnostics, - ...(!teamActive ? ['team_runtime_inactive'] : []), - ...decision.diagnostics, - ], + diagnostics: [...agenda.diagnostics, ...runtimeActivity.diagnostics, ...decision.diagnostics], ...(source.providerId ? { providerId: source.providerId } : {}), }); + assertReconcileNotCancelled(context); await this.deps.statusStore.write(status); + assertReconcileNotCancelled(context); await this.planNudgeOutbox(status); return status; } diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts b/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts index 8899468f..fb23c682 100644 --- a/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts +++ b/src/features/member-work-sync/core/application/MemberWorkSyncReporter.ts @@ -6,6 +6,7 @@ import { finalizeMemberWorkSyncAgenda, MemberWorkSyncReconciler, } from './MemberWorkSyncReconciler'; +import { resolveMemberWorkSyncRuntimeActivity } from './MemberWorkSyncRuntimeActivity'; import type { MemberWorkSyncReport, @@ -42,10 +43,11 @@ export class MemberWorkSyncReporter { const source = await this.deps.agendaSource.loadAgenda(request); const agenda = finalizeMemberWorkSyncAgenda(this.deps, source); const nowIso = this.deps.clock.now().toISOString(); - const teamActive = this.deps.lifecycle - ? await this.deps.lifecycle.isTeamActive(agenda.teamName) - : true; - if (!teamActive) { + const runtimeActivity = await resolveMemberWorkSyncRuntimeActivity(this.deps, { + teamName: agenda.teamName, + memberName: agenda.memberName, + }); + if (!runtimeActivity.teamActive) { const status = await this.reconciler.execute(request); const rejectedStatus = await this.recordRejectedReport( status, @@ -59,6 +61,21 @@ export class MemberWorkSyncReporter { status: rejectedStatus, }; } + if (!runtimeActivity.memberActive) { + const status = await this.reconciler.execute(request); + const rejectedStatus = await this.recordRejectedReport( + status, + request, + 'member_runtime_inactive' + ); + return { + accepted: false, + code: 'member_runtime_inactive', + message: + 'Member runtime is not active. Restart this teammate before reporting work sync state.', + status: rejectedStatus, + }; + } const tokenValidation = this.deps.reportToken ? await this.deps.reportToken.verify({ token: request.reportToken, diff --git a/src/features/member-work-sync/core/application/MemberWorkSyncRuntimeActivity.ts b/src/features/member-work-sync/core/application/MemberWorkSyncRuntimeActivity.ts new file mode 100644 index 00000000..91ee0e9f --- /dev/null +++ b/src/features/member-work-sync/core/application/MemberWorkSyncRuntimeActivity.ts @@ -0,0 +1,41 @@ +import type { MemberWorkSyncUseCaseDeps } from './ports'; + +export interface MemberWorkSyncRuntimeActivity { + teamActive: boolean; + memberActive: boolean; + inactive: boolean; + diagnostics: string[]; +} + +export async function resolveMemberWorkSyncRuntimeActivity( + deps: Pick, + input: { teamName: string; memberName: string } +): Promise { + if (!deps.lifecycle) { + return { teamActive: true, memberActive: true, inactive: false, diagnostics: [] }; + } + + const teamActive = await deps.lifecycle.isTeamActive(input.teamName); + if (!teamActive) { + return { + teamActive: false, + memberActive: false, + inactive: true, + diagnostics: ['team_runtime_inactive'], + }; + } + + const memberActive = deps.lifecycle.isMemberActive + ? await deps.lifecycle.isMemberActive(input) + : true; + if (!memberActive) { + return { + teamActive: true, + memberActive: false, + inactive: true, + diagnostics: ['member_runtime_inactive'], + }; + } + + return { teamActive: true, memberActive: true, inactive: false, diagnostics: [] }; +} diff --git a/src/features/member-work-sync/core/application/index.ts b/src/features/member-work-sync/core/application/index.ts index 8232db8a..1704cb76 100644 --- a/src/features/member-work-sync/core/application/index.ts +++ b/src/features/member-work-sync/core/application/index.ts @@ -8,6 +8,7 @@ export * from './MemberWorkSyncNudgeOutboxPlanner'; export * from './MemberWorkSyncPendingReportIntentReplayer'; export * from './MemberWorkSyncReconciler'; export * from './MemberWorkSyncReporter'; +export * from './MemberWorkSyncRuntimeActivity'; export * from './MemberWorkSyncTargetedRecoveryPolicy'; export type * from './ports'; export * from './RuntimeTurnSettledIngestor'; diff --git a/src/features/member-work-sync/core/application/ports.ts b/src/features/member-work-sync/core/application/ports.ts index 64e2c4fa..58400bce 100644 --- a/src/features/member-work-sync/core/application/ports.ts +++ b/src/features/member-work-sync/core/application/ports.ts @@ -56,6 +56,7 @@ export interface MemberWorkSyncReportTokenPort { export interface MemberWorkSyncLifecyclePort { isTeamActive(teamName: string): Promise | boolean; + isMemberActive?(input: { teamName: string; memberName: string }): Promise | boolean; } export interface MemberWorkSyncLoggerPort { @@ -71,6 +72,7 @@ export type MemberWorkSyncAuditEventName = | 'turn_settled_ignored' | 'queue_enqueued' | 'queue_coalesced' + | 'queue_retry_scheduled' | 'queue_reconciled' | 'queue_dropped' | 'reconcile_started' @@ -197,6 +199,12 @@ export interface MemberWorkSyncWatchdogCooldownPort { taskIds: string[]; nowIso: string; }): Promise; + getRecentNudgeCooldown?(input: { + teamName: string; + memberName: string; + taskIds: string[]; + nowIso: string; + }): Promise<{ active: boolean; retryAfterIso?: string }>; } export interface MemberWorkSyncBusySignalPort { diff --git a/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts b/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts index d934a14b..641220bf 100644 --- a/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts +++ b/src/features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter.ts @@ -77,6 +77,7 @@ export class MemberWorkSyncTeamChangeRouter { noteTeamChange(event: TeamChangeEvent): void { if (event.type === 'lead-activity' && event.detail === 'offline') { this.queue.dropTeam(event.teamName); + void this.enqueueTeam(event.teamName, 'runtime_activity', 0).catch(() => undefined); return; } diff --git a/src/features/member-work-sync/main/adapters/output/TeamTaskStallJournalWorkSyncCooldown.ts b/src/features/member-work-sync/main/adapters/output/TeamTaskStallJournalWorkSyncCooldown.ts index 1873ee06..c5dd44b4 100644 --- a/src/features/member-work-sync/main/adapters/output/TeamTaskStallJournalWorkSyncCooldown.ts +++ b/src/features/member-work-sync/main/adapters/output/TeamTaskStallJournalWorkSyncCooldown.ts @@ -7,10 +7,13 @@ const DEFAULT_WATCHDOG_COOLDOWN_MS = 10 * 60_000; interface StallJournalEntry { taskId: string; + memberName?: string; state: string; alertedAt?: string; } +type WatchdogCooldownResult = { active: boolean; retryAfterIso?: string }; + function parseTime(value: string | undefined): number | null { if (!value) { return null; @@ -19,6 +22,10 @@ function parseTime(value: string | undefined): number | null { return Number.isFinite(time) ? time : null; } +function normalizeMemberName(value: string | undefined): string { + return value?.trim().toLowerCase() ?? ''; +} + export class TeamTaskStallJournalWorkSyncCooldown implements MemberWorkSyncWatchdogCooldownPort { constructor( private readonly teamsBasePath: string, @@ -31,9 +38,18 @@ export class TeamTaskStallJournalWorkSyncCooldown implements MemberWorkSyncWatch taskIds: string[]; nowIso: string; }): Promise { + return (await this.getRecentNudgeCooldown(input)).active; + } + + async getRecentNudgeCooldown(input: { + teamName: string; + memberName: string; + taskIds: string[]; + nowIso: string; + }): Promise { const taskIds = new Set(input.taskIds); if (taskIds.size === 0) { - return false; + return { active: false }; } try { @@ -43,19 +59,34 @@ export class TeamTaskStallJournalWorkSyncCooldown implements MemberWorkSyncWatch ); const parsed = JSON.parse(raw) as unknown; if (!Array.isArray(parsed)) { - return false; + return { active: false }; } const now = parseTime(input.nowIso) ?? Date.now(); - return parsed.some((entry): boolean => { + const expectedMemberName = normalizeMemberName(input.memberName); + let retryAfterMs: number | null = null; + for (const entry of parsed) { const row = entry as Partial; if (row.state !== 'alerted' || !row.taskId || !taskIds.has(row.taskId)) { - return false; + continue; + } + const rowMemberName = normalizeMemberName(row.memberName); + if (rowMemberName && rowMemberName !== expectedMemberName) { + continue; } const alertedAt = parseTime(row.alertedAt); - return alertedAt != null && now - alertedAt <= this.cooldownMs; - }); + if (alertedAt == null || alertedAt > now || now - alertedAt >= this.cooldownMs) { + continue; + } + const entryRetryAfterMs = alertedAt + this.cooldownMs; + retryAfterMs = + retryAfterMs == null ? entryRetryAfterMs : Math.max(retryAfterMs, entryRetryAfterMs); + } + if (retryAfterMs == null) { + return { active: false }; + } + return { active: true, retryAfterIso: new Date(retryAfterMs).toISOString() }; } catch { - return false; + return { active: false }; } } } diff --git a/src/features/member-work-sync/main/composition/__tests__/memberWorkSyncTeamActivity.test.ts b/src/features/member-work-sync/main/composition/__tests__/memberWorkSyncTeamActivity.test.ts index d352c19f..7079c5e2 100644 --- a/src/features/member-work-sync/main/composition/__tests__/memberWorkSyncTeamActivity.test.ts +++ b/src/features/member-work-sync/main/composition/__tests__/memberWorkSyncTeamActivity.test.ts @@ -1,8 +1,11 @@ import { describe, expect, it } from 'vitest'; import { + hasUncertainWorkSyncRuntimeActivity, hasWorkSyncActiveRuntime, isRuntimeEntryActiveForWorkSync, + isRuntimeMemberActiveForWorkSync, + isRuntimeMemberActivityUncertainForWorkSync, } from '../memberWorkSyncTeamActivity'; import type { TeamAgentRuntimeEntry, TeamAgentRuntimeSnapshot } from '@shared/types'; @@ -39,14 +42,49 @@ describe('member work sync team activity', () => { }); it('treats a confirmed bootstrap runtime entry as active', () => { + for (const pidSource of ['agent_process_table', 'opencode_bridge'] as const) { + expect( + isRuntimeEntryActiveForWorkSync( + createRuntimeEntry({ + livenessKind: 'confirmed_bootstrap', + pidSource, + runtimeLastSeenAt: '2026-05-18T19:44:47.000Z', + }) + ) + ).toBe(true); + } + }); + + it('does not treat bootstrap-only confirmation as active runtime evidence', () => { + for (const pidSource of [ + undefined, + 'runtime_bootstrap', + 'persisted_metadata', + 'tmux_child', + 'tmux_pane', + ] as const) { + expect( + isRuntimeEntryActiveForWorkSync( + createRuntimeEntry({ + livenessKind: 'confirmed_bootstrap', + ...(pidSource ? { pidSource } : {}), + }) + ) + ).toBe(false); + } + }); + + it('does not count lead runtime entries as work-sync active teammates', () => { expect( isRuntimeEntryActiveForWorkSync( createRuntimeEntry({ - livenessKind: 'confirmed_bootstrap', - runtimeLastSeenAt: '2026-05-18T19:44:47.000Z', + memberName: 'team-lead', + backendType: 'lead', + livenessKind: undefined, + pidSource: 'lead_process', }) ) - ).toBe(true); + ).toBe(false); }); it('does not treat inactive liveness diagnostics as active by themselves', () => { @@ -77,6 +115,12 @@ describe('member work sync team activity', () => { expect( hasWorkSyncActiveRuntime( createRuntimeSnapshot({ + 'team-lead': createRuntimeEntry({ + memberName: 'team-lead', + backendType: 'lead', + livenessKind: undefined, + pidSource: 'lead_process', + }), alice: createRuntimeEntry({ alive: false, livenessKind: 'stale_metadata' }), bob: createRuntimeEntry({ memberName: 'bob', livenessKind: 'runtime_process' }), }) @@ -88,6 +132,12 @@ describe('member work sync team activity', () => { expect( hasWorkSyncActiveRuntime( createRuntimeSnapshot({ + 'team-lead': createRuntimeEntry({ + memberName: 'team-lead', + backendType: 'lead', + livenessKind: undefined, + pidSource: 'lead_process', + }), alice: createRuntimeEntry({ alive: false, livenessKind: 'stale_metadata' }), bob: createRuntimeEntry({ memberName: 'bob', @@ -99,6 +149,50 @@ describe('member work sync team activity', () => { ).toBe(false); }); + it('checks active runtime evidence for a specific teammate', () => { + const snapshot = createRuntimeSnapshot({ + alice: createRuntimeEntry({ memberName: 'alice', livenessKind: 'runtime_process' }), + bob: createRuntimeEntry({ memberName: 'bob', alive: false, livenessKind: 'stale_metadata' }), + }); + + expect(isRuntimeMemberActiveForWorkSync(snapshot, 'ALICE')).toBe(true); + expect(isRuntimeMemberActiveForWorkSync(snapshot, 'bob')).toBe(false); + expect(isRuntimeMemberActiveForWorkSync(snapshot, 'team-lead')).toBe(false); + }); + + it('treats process table unavailability as uncertain runtime activity', () => { + const snapshot = createRuntimeSnapshot({ + alice: createRuntimeEntry({ + memberName: 'alice', + alive: false, + livenessKind: 'registered_only', + runtimeDiagnostic: 'runtime pid could not be verified because process table unavailable', + }), + bob: createRuntimeEntry({ memberName: 'bob', alive: false, livenessKind: 'stale_metadata' }), + }); + + expect(hasWorkSyncActiveRuntime(snapshot)).toBe(false); + expect(hasUncertainWorkSyncRuntimeActivity(snapshot)).toBe(true); + expect(isRuntimeMemberActivityUncertainForWorkSync(snapshot, 'alice')).toBe(true); + expect(isRuntimeMemberActivityUncertainForWorkSync(snapshot, 'bob')).toBe(false); + }); + + it('recognizes process table is unavailable diagnostics as uncertain runtime activity', () => { + const snapshot = createRuntimeSnapshot({ + alice: createRuntimeEntry({ + memberName: 'alice', + alive: false, + livenessKind: 'confirmed_bootstrap', + pidSource: 'runtime_bootstrap', + runtimeDiagnostic: 'runtime pid could not be verified because process table is unavailable', + }), + }); + + expect(hasWorkSyncActiveRuntime(snapshot)).toBe(false); + expect(hasUncertainWorkSyncRuntimeActivity(snapshot)).toBe(true); + expect(isRuntimeMemberActivityUncertainForWorkSync(snapshot, 'alice')).toBe(true); + }); + it('handles missing snapshots as inactive', () => { expect(hasWorkSyncActiveRuntime(null)).toBe(false); expect(hasWorkSyncActiveRuntime(undefined)).toBe(false); diff --git a/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts b/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts index ecd2e932..c617e653 100644 --- a/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts +++ b/src/features/member-work-sync/main/composition/createMemberWorkSyncFeature.ts @@ -228,6 +228,7 @@ export function createMemberWorkSyncFeature(deps: { kanbanManager: TeamKanbanManager; membersMetaStore: TeamMembersMetaStore; isTeamActive?: (teamName: string) => Promise | boolean; + isMemberActive?: (input: { teamName: string; memberName: string }) => Promise | boolean; canDispatchNudges?: (teamName: string) => Promise | boolean; listLifecycleActiveTeamNames?: () => Promise; queueQuietWindowMs?: number; @@ -312,7 +313,14 @@ export function createMemberWorkSyncFeature(deps: { ...(deps.reviewPickupEscalation ? { reviewPickupEscalation: deps.reviewPickupEscalation } : {}), reportToken, auditJournal, - ...(deps.isTeamActive ? { lifecycle: { isTeamActive: deps.isTeamActive } } : {}), + ...(deps.isTeamActive + ? { + lifecycle: { + isTeamActive: deps.isTeamActive, + ...(deps.isMemberActive ? { isMemberActive: deps.isMemberActive } : {}), + }, + } + : {}), logger: deps.logger, }; const diagnosticsReader = new MemberWorkSyncDiagnosticsReader(useCaseDeps); @@ -328,6 +336,16 @@ export function createMemberWorkSyncFeature(deps: { retryable: 0, terminal: 0, }); + const addNudgeDispatchSummaries = ( + left: MemberWorkSyncNudgeDispatchSummary, + right: MemberWorkSyncNudgeDispatchSummary + ): MemberWorkSyncNudgeDispatchSummary => ({ + claimed: left.claimed + right.claimed, + delivered: left.delivered + right.delivered, + superseded: left.superseded + right.superseded, + retryable: left.retryable + right.retryable, + terminal: left.terminal + right.terminal, + }); const filterNudgeDispatchReadyTeamNames = async (teamNames: string[]): Promise => { const uniqueTeamNames = [...new Set(teamNames.map((name) => name.trim()).filter(Boolean))]; if (!deps.canDispatchNudges) { @@ -401,22 +419,30 @@ export function createMemberWorkSyncFeature(deps: { if (readyTeamNames.length === 0) { return emptyNudgeDispatchSummary(); } + const dispatchReadyNudges = () => + nudgeDispatcher.dispatchDue({ + teamNames: readyTeamNames, + claimedBy, + }); + const initialSummary = await dispatchReadyNudges(); if (options.refreshBackgroundStaleStatuses !== false) { await refreshBackgroundStaleStatuses(readyTeamNames); + return addNudgeDispatchSummaries(initialSummary, await dispatchReadyNudges()); } - return nudgeDispatcher.dispatchDue({ - teamNames: readyTeamNames, - claimedBy, - }); + return initialSummary; }; const queue = new MemberWorkSyncEventQueue({ reconcile: async (request, context: MemberWorkSyncReconcileContext) => { await reconciler.execute(request, context); + if (context.isCancelled?.()) { + return; + } await dispatchNudgesForReadyTeams([request.teamName], `member-work-sync:${process.pid}`, { refreshBackgroundStaleStatuses: false, }); }, isTeamActive: deps.isTeamActive ?? (() => true), + reconcileInactiveTeams: true, ...(deps.queueQuietWindowMs != null ? { quietWindowMs: deps.queueQuietWindowMs } : {}), auditJournal, logger: deps.logger, diff --git a/src/features/member-work-sync/main/composition/memberWorkSyncTeamActivity.ts b/src/features/member-work-sync/main/composition/memberWorkSyncTeamActivity.ts index a6a475a2..a739f201 100644 --- a/src/features/member-work-sync/main/composition/memberWorkSyncTeamActivity.ts +++ b/src/features/member-work-sync/main/composition/memberWorkSyncTeamActivity.ts @@ -1,7 +1,17 @@ -import type { TeamAgentRuntimeEntry, TeamAgentRuntimeSnapshot } from '@shared/types'; +import { mentionsProcessTableUnavailable } from '@shared/utils/teamLaunchFailureReason'; + +import { normalizeMemberName } from '../../core/domain'; + +import type { + TeamAgentRuntimeEntry, + TeamAgentRuntimePidSource, + TeamAgentRuntimeSnapshot, +} from '@shared/types'; type RuntimeLivenessKind = NonNullable; +const WORK_SYNC_RESERVED_MEMBER_NAMES = new Set(['team-lead', 'user']); + const WORK_SYNC_INACTIVE_LIVENESS_KINDS = new Set([ 'permission_blocked', 'runtime_process_candidate', @@ -11,20 +21,107 @@ const WORK_SYNC_INACTIVE_LIVENESS_KINDS = new Set([ 'not_found', ]); +const WORK_SYNC_BOOTSTRAP_ONLY_PID_SOURCES = new Set([ + 'runtime_bootstrap', + 'persisted_metadata', +]); + +const WORK_SYNC_CONFIRMED_BOOTSTRAP_ACTIVE_PID_SOURCES = new Set([ + 'agent_process_table', + 'opencode_bridge', +]); + export function isRuntimeEntryActiveForWorkSync( - entry: Pick | null | undefined + entry: + | Pick< + TeamAgentRuntimeEntry, + 'alive' | 'backendType' | 'livenessKind' | 'memberName' | 'pidSource' + > + | null + | undefined ): boolean { if (entry?.alive !== true) { return false; } + if ( + entry.backendType === 'lead' || + WORK_SYNC_RESERVED_MEMBER_NAMES.has(entry.memberName.trim().toLowerCase()) + ) { + return false; + } + if ( + entry.livenessKind === 'confirmed_bootstrap' && + (!entry.pidSource || + WORK_SYNC_BOOTSTRAP_ONLY_PID_SOURCES.has(entry.pidSource) || + !WORK_SYNC_CONFIRMED_BOOTSTRAP_ACTIVE_PID_SOURCES.has(entry.pidSource)) + ) { + return false; + } if (!entry.livenessKind) { return true; } return !WORK_SYNC_INACTIVE_LIVENESS_KINDS.has(entry.livenessKind); } +function isRuntimeEntryRelevantForWorkSync( + entry: Pick +): boolean { + return ( + entry.backendType !== 'lead' && + !WORK_SYNC_RESERVED_MEMBER_NAMES.has(entry.memberName.trim().toLowerCase()) + ); +} + +function runtimeEntryMentionsProcessTableUnavailable( + entry: Pick +): boolean { + return [entry.runtimeDiagnostic, ...(entry.diagnostics ?? [])].some((message) => + mentionsProcessTableUnavailable(message) + ); +} + +export function hasUncertainWorkSyncRuntimeActivity( + snapshot: Pick | null | undefined +): boolean { + return Object.values(snapshot?.members ?? {}).some( + (entry) => + isRuntimeEntryRelevantForWorkSync(entry) && runtimeEntryMentionsProcessTableUnavailable(entry) + ); +} + export function hasWorkSyncActiveRuntime( snapshot: Pick | null | undefined ): boolean { return Object.values(snapshot?.members ?? {}).some(isRuntimeEntryActiveForWorkSync); } + +export function isRuntimeMemberActiveForWorkSync( + snapshot: Pick | null | undefined, + memberName: string +): boolean { + const normalizedMemberName = normalizeMemberName(memberName); + if (!normalizedMemberName) { + return false; + } + return Object.values(snapshot?.members ?? {}).some( + (entry) => + normalizeMemberName(entry.memberName) === normalizedMemberName && + isRuntimeEntryActiveForWorkSync(entry) + ); +} + +export function isRuntimeMemberActivityUncertainForWorkSync( + snapshot: Pick | null | undefined, + memberName: string +): boolean { + const normalizedMemberName = normalizeMemberName(memberName); + if (!normalizedMemberName) { + return false; + } + return Object.values(snapshot?.members ?? {}).some( + (entry) => + normalizeMemberName(entry.memberName) === normalizedMemberName && + isRuntimeEntryRelevantForWorkSync(entry) && + runtimeEntryMentionsProcessTableUnavailable(entry) + ); +} diff --git a/src/features/member-work-sync/main/index.ts b/src/features/member-work-sync/main/index.ts index 6f5f1fa8..f451680f 100644 --- a/src/features/member-work-sync/main/index.ts +++ b/src/features/member-work-sync/main/index.ts @@ -9,6 +9,9 @@ export { createMemberWorkSyncFeature, } from './composition/createMemberWorkSyncFeature'; export { + hasUncertainWorkSyncRuntimeActivity, hasWorkSyncActiveRuntime, isRuntimeEntryActiveForWorkSync, + isRuntimeMemberActiveForWorkSync, + isRuntimeMemberActivityUncertainForWorkSync, } from './composition/memberWorkSyncTeamActivity'; diff --git a/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts b/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts index 4904265a..c1477ed0 100644 --- a/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts +++ b/src/features/member-work-sync/main/infrastructure/JsonMemberWorkSyncStore.ts @@ -110,6 +110,7 @@ interface OutboxIndexFile { type OutboxIndexRoute = OutboxIndexFile['items'][string]; type OutboxDueRoute = [string, OutboxIndexRoute]; +const MEMBER_WORK_SYNC_OUTBOX_CLAIM_STALE_MS = 5 * 60 * 1000; export interface JsonMemberWorkSyncStoreDeps { auditJournal?: MemberWorkSyncAuditJournalPort; @@ -117,8 +118,12 @@ export interface JsonMemberWorkSyncStoreDeps { now?: () => Date; } -function normalizeMemberKey(memberName: string): string { - return memberName.trim().toLowerCase(); +function normalizeMemberKey(memberName: unknown): string { + return typeof memberName === 'string' ? memberName.trim().toLowerCase() : ''; +} + +function normalizeTeamKey(teamName: unknown): string { + return typeof teamName === 'string' ? teamName.trim().toLowerCase() : ''; } function emptyMetricsIndex(): MetricsIndexFile { @@ -242,6 +247,46 @@ function canReviveOutboxItem(status: MemberWorkSyncOutboxItem['status']): boolea return status === 'superseded' || (!isOutboxTerminal(status) && status !== 'pending'); } +function isReportIntentOwnedBy( + teamName: string, + memberName: string, + intent: MemberWorkSyncReportIntent +): boolean { + return ( + normalizeTeamKey(intent.teamName) === normalizeTeamKey(teamName) && + normalizeMemberKey(intent.memberName) === normalizeMemberKey(memberName) + ); +} + +function isOutboxItemOwnedBy( + teamName: string, + memberName: string, + item: MemberWorkSyncOutboxItem +): boolean { + return ( + normalizeTeamKey(item.teamName) === normalizeTeamKey(teamName) && + normalizeMemberKey(item.memberName) === normalizeMemberKey(memberName) + ); +} + +function parseIsoMs(value: string | undefined): number | null { + if (!value) { + return null; + } + const ms = Date.parse(value); + return Number.isFinite(ms) ? ms : null; +} + +function isStaleClaim(claimedAt: string | undefined, nowIso: string): boolean { + const claimedAtMs = parseIsoMs(claimedAt); + const nowMs = parseIsoMs(nowIso); + return ( + claimedAtMs != null && + nowMs != null && + (claimedAtMs > nowMs || nowMs - claimedAtMs >= MEMBER_WORK_SYNC_OUTBOX_CLAIM_STALE_MS) + ); +} + function applyOptionalNextAttemptAt( item: MemberWorkSyncOutboxItem, nextAttemptAt: string | undefined @@ -253,14 +298,36 @@ function applyOptionalNextAttemptAt( delete item.nextAttemptAt; } +function isNextAttemptDue(nextAttemptAt: string | undefined, nowIso: string): boolean { + if (!nextAttemptAt) { + return true; + } + const nextAttemptAtMs = parseIsoMs(nextAttemptAt); + if (nextAttemptAtMs == null) { + return true; + } + const nowMs = parseIsoMs(nowIso); + return nowMs != null && nextAttemptAtMs <= nowMs; +} + function canClaimOutboxItem(item: MemberWorkSyncOutboxItem, nowIso: string): boolean { + if (item.status === 'claimed') { + return isStaleClaim(item.claimedAt ?? item.updatedAt, nowIso); + } if (item.status !== 'pending' && item.status !== 'failed_retryable') { return false; } - if (!item.nextAttemptAt) { - return true; + return isNextAttemptDue(item.nextAttemptAt, nowIso); +} + +function canClaimOutboxRoute(route: OutboxIndexRoute, nowIso: string): boolean { + if (route.status === 'claimed') { + return isStaleClaim(route.updatedAt, nowIso); } - return item.nextAttemptAt <= nowIso; + return ( + (route.status === 'pending' || route.status === 'failed_retryable') && + isNextAttemptDue(route.nextAttemptAt, nowIso) + ); } function getDueOutboxRoutes( @@ -269,8 +336,7 @@ function getDueOutboxRoutes( limit: number ): OutboxDueRoute[] { return Object.entries(index.items) - .filter(([, route]) => route.status === 'pending' || route.status === 'failed_retryable') - .filter(([, route]) => !route.nextAttemptAt || route.nextAttemptAt <= nowIso) + .filter(([, route]) => canClaimOutboxRoute(route, nowIso)) .sort((left, right) => { const leftTime = left[1].nextAttemptAt ?? left[1].updatedAt; const rightTime = right[1].nextAttemptAt ?? right[1].updatedAt; @@ -623,10 +689,10 @@ export class JsonMemberWorkSyncStore staleIndex = true; } } - const missingIndexedPending = staleIndex + const unindexedOrStaleIndexedPending = staleIndex ? false - : await this.hasMissingIndexedPendingReport(teamName, index); - if (staleIndex || missingIndexedPending) { + : await this.hasUnindexedOrStaleIndexedPendingReport(teamName, index); + if (staleIndex || unindexedOrStaleIndexedPending) { await this.enqueue(teamName, async () => { await withFileLock(this.paths.getPendingReportsIndexPath(teamName), async () => { index = await this.repairPendingReportsIndex(teamName); @@ -666,29 +732,58 @@ export class JsonMemberWorkSyncStore if (!route) { return; } - await withFileLock( - this.paths.getMemberReportsPath(teamName, route.memberName), - async () => { - const reports = await this.readMemberReportsFile(teamName, route.memberName); - const current = reports.intents[id]; - if (current?.status !== 'pending') { - return; + const updateRoute = async ( + targetRoute: PendingReportsIndexFile['items'][string] + ): Promise => { + let staleRoute = false; + await withFileLock( + this.paths.getMemberReportsPath(teamName, targetRoute.memberName), + async () => { + const reports = await this.readMemberReportsFile(teamName, targetRoute.memberName); + const current = reports.intents[id]; + if (!current) { + delete index.items[id]; + staleRoute = true; + return; + } + if (!isReportIntentOwnedBy(teamName, targetRoute.memberName, current)) { + delete index.items[id]; + staleRoute = true; + return; + } + if (current.status !== 'pending') { + return; + } + const next: MemberWorkSyncReportIntent = { + ...current, + status: result.status, + resultCode: result.resultCode, + processedAt: result.processedAt, + }; + reports.intents[id] = next; + await this.writeMemberReportsFile(teamName, targetRoute.memberName, reports); + index.items[id] = toPendingReportIndexItem( + next, + this.paths.getMemberKey(next.memberName) + ); + await this.writePendingReportsIndexFile(teamName, index); } - reports.intents[id] = { - ...current, - status: result.status, - resultCode: result.resultCode, - processedAt: result.processedAt, - }; - await this.writeMemberReportsFile(teamName, route.memberName, reports); - index.items[id] = { - ...route, - status: result.status, - processedAt: result.processedAt, - }; - await this.writePendingReportsIndexFile(teamName, index); + ); + return staleRoute; + }; + + let staleRoute = await updateRoute(route); + if (staleRoute) { + index = await this.repairPendingReportsIndex(teamName); + const repairedRoute = index.items[id]; + if (!repairedRoute) { + return; } - ); + staleRoute = await updateRoute(repairedRoute); + if (staleRoute) { + await this.repairPendingReportsIndex(teamName); + } + } }); }); } @@ -801,45 +896,67 @@ export class JsonMemberWorkSyncStore } let dueRoutes = getDueOutboxRoutes(index, input.nowIso, input.limit); if ( - dueRoutes.length > 0 && dueRoutes.length < Math.max(0, input.limit) && - (await this.hasMissingIndexedDueOutboxItem(input.teamName, index, input.nowIso)) + (await this.hasUnindexedOrStaleIndexedDueOutboxItem(input.teamName, index, input.nowIso)) ) { index = await this.repairOutboxIndex(input.teamName); dueRoutes = getDueOutboxRoutes(index, input.nowIso, input.limit); } - let staleIndex = false; - for (const [id, route] of dueRoutes) { - await withFileLock( - this.paths.getMemberOutboxPath(input.teamName, route.memberName), - async () => { - const outbox = await this.readMemberOutboxFile(input.teamName, route.memberName); - const item = outbox.items[id]; - if (!item || !canClaimOutboxItem(item, input.nowIso)) { - delete index.items[id]; - staleIndex = true; - return; - } - const next: MemberWorkSyncOutboxItem = { - ...item, - status: 'claimed', - attemptGeneration: item.attemptGeneration + 1, - claimedBy: input.claimedBy, - claimedAt: input.nowIso, - updatedAt: input.nowIso, - }; - delete next.lastError; - outbox.items[id] = next; - await this.writeMemberOutboxFile(input.teamName, route.memberName, outbox); - index.items[id] = toOutboxIndexItem(next, route.memberKey); - claimed.push(next); + const claimRoutes = async (routes: OutboxDueRoute[]): Promise => { + let staleIndex = false; + for (const [id, route] of routes) { + if (claimed.length >= Math.max(0, input.limit)) { + break; } - ); - } + await withFileLock( + this.paths.getMemberOutboxPath(input.teamName, route.memberName), + async () => { + const outbox = await this.readMemberOutboxFile(input.teamName, route.memberName); + const item = outbox.items[id]; + if (!item || !canClaimOutboxItem(item, input.nowIso)) { + delete index.items[id]; + staleIndex = true; + return; + } + const memberKey = this.paths.getMemberKey(item.memberName); + if (!isOutboxItemOwnedBy(input.teamName, route.memberName, item)) { + delete index.items[id]; + staleIndex = true; + return; + } + const next: MemberWorkSyncOutboxItem = { + ...item, + status: 'claimed', + attemptGeneration: item.attemptGeneration + 1, + claimedBy: input.claimedBy, + claimedAt: input.nowIso, + updatedAt: input.nowIso, + }; + delete next.nextAttemptAt; + delete next.lastError; + outbox.items[id] = next; + await this.writeMemberOutboxFile(input.teamName, route.memberName, outbox); + index.items[id] = toOutboxIndexItem(next, memberKey); + claimed.push(next); + } + ); + } + return staleIndex; + }; + let staleIndex = await claimRoutes(dueRoutes); if (staleIndex) { index = await this.repairOutboxIndex(input.teamName); + const remainingLimit = Math.max(0, input.limit) - claimed.length; + dueRoutes = + remainingLimit > 0 ? getDueOutboxRoutes(index, input.nowIso, remainingLimit) : []; + staleIndex = dueRoutes.length > 0 ? await claimRoutes(dueRoutes) : false; + if (staleIndex) { + await this.repairOutboxIndex(input.teamName); + } else if (dueRoutes.length > 0) { + await this.writeOutboxIndexFile(input.teamName, index); + } } else if (dueRoutes.length > 0) { await this.writeOutboxIndexFile(input.teamName, index); } @@ -850,7 +967,7 @@ export class JsonMemberWorkSyncStore async markDelivered(input: MemberWorkSyncOutboxMarkDeliveredInput): Promise { await this.updateOutboxItem(input.teamName, input.id, (current) => { - if (current?.attemptGeneration !== input.attemptGeneration) { + if (current?.attemptGeneration !== input.attemptGeneration || current.status !== 'claimed') { return current; } const next: MemberWorkSyncOutboxItem = { @@ -885,7 +1002,10 @@ export class JsonMemberWorkSyncStore async markFailed(input: MemberWorkSyncOutboxMarkFailedInput): Promise { await this.updateOutboxItem(input.teamName, input.id, (current) => { - if (current?.attemptGeneration !== input.attemptGeneration) { + if ( + current?.attemptGeneration !== input.attemptGeneration || + isOutboxTerminal(current.status) + ) { return current; } const next: MemberWorkSyncOutboxItem = { @@ -996,7 +1116,8 @@ export class JsonMemberWorkSyncStore (item) => item.payload.workSyncIntentKey === intentKey && item.updatedAt >= input.sinceIso && - item.status !== 'failed_terminal' + item.status !== 'failed_terminal' && + item.status !== 'superseded' ) .sort((left, right) => right.updatedAt.localeCompare(left.updatedAt)); const latest = matches[0]; @@ -1171,17 +1292,48 @@ export class JsonMemberWorkSyncStore if (!route) { return; } - await withFileLock(this.paths.getMemberOutboxPath(teamName, route.memberName), async () => { - const outbox = await this.readMemberOutboxFile(teamName, route.memberName); - const next = updater(outbox.items[id]); - if (!next) { + const updateRoute = async (targetRoute: OutboxIndexRoute): Promise => { + let staleRoute = false; + await withFileLock( + this.paths.getMemberOutboxPath(teamName, targetRoute.memberName), + async () => { + const outbox = await this.readMemberOutboxFile(teamName, targetRoute.memberName); + const current = outbox.items[id]; + if (!current) { + delete index.items[id]; + staleRoute = true; + return; + } + if (!isOutboxItemOwnedBy(teamName, targetRoute.memberName, current)) { + delete index.items[id]; + staleRoute = true; + return; + } + const next = updater(current); + if (!next) { + return; + } + outbox.items[id] = next; + await this.writeMemberOutboxFile(teamName, targetRoute.memberName, outbox); + index.items[id] = toOutboxIndexItem(next, this.paths.getMemberKey(next.memberName)); + await this.writeOutboxIndexFile(teamName, index); + } + ); + return staleRoute; + }; + + let staleRoute = await updateRoute(route); + if (staleRoute) { + index = await this.repairOutboxIndex(teamName); + const repairedRoute = index.items[id]; + if (!repairedRoute) { return; } - outbox.items[id] = next; - await this.writeMemberOutboxFile(teamName, route.memberName, outbox); - index.items[id] = toOutboxIndexItem(next, route.memberKey); - await this.writeOutboxIndexFile(teamName, index); - }); + staleRoute = await updateRoute(repairedRoute); + if (staleRoute) { + await this.repairOutboxIndex(teamName); + } + } }); }); } @@ -1251,11 +1403,17 @@ export class JsonMemberWorkSyncStore for (const { memberName, reports } of await this.scanMemberReports(teamName)) { const memberKey = this.paths.getMemberKey(memberName); for (const intent of Object.values(reports.intents)) { + if (!isReportIntentOwnedBy(teamName, memberName, intent)) { + continue; + } index.items[intent.id] = toPendingReportIndexItem(intent, memberKey); repairedMembers.add(intent.memberName); } } for (const intent of Object.values((await this.readLegacyPendingFile(teamName)).intents)) { + if (!isReportIntentOwnedBy(teamName, intent.memberName, intent)) { + continue; + } const memberKey = this.paths.getMemberKey(intent.memberName); if (!index.items[intent.id]) { await withFileLock( @@ -1300,11 +1458,17 @@ export class JsonMemberWorkSyncStore for (const { memberName, outbox } of await this.scanMemberOutboxes(teamName)) { const memberKey = this.paths.getMemberKey(memberName); for (const item of Object.values(outbox.items)) { + if (!isOutboxItemOwnedBy(teamName, memberName, item)) { + continue; + } index.items[item.id] = toOutboxIndexItem(item, memberKey); repairedMembers.add(item.memberName); } } for (const item of Object.values((await this.readLegacyOutboxFile(teamName)).items)) { + if (!isOutboxItemOwnedBy(teamName, item.memberName, item)) { + continue; + } const memberKey = this.paths.getMemberKey(item.memberName); if (!index.items[item.id]) { await withFileLock(this.paths.getMemberOutboxPath(teamName, item.memberName), async () => { @@ -1382,26 +1546,54 @@ export class JsonMemberWorkSyncStore return reports; } - private async hasMissingIndexedPendingReport( + private async hasUnindexedOrStaleIndexedPendingReport( teamName: string, index: PendingReportsIndexFile ): Promise { - const indexedIds = new Set(Object.keys(index.items)); - for (const { reports } of await this.scanMemberReports(teamName)) { + const routes = index.items; + for (const { memberName, reports } of await this.scanMemberReports(teamName)) { for (const intent of Object.values(reports.intents)) { - if (intent.status === 'pending' && !indexedIds.has(intent.id)) { + if (!isReportIntentOwnedBy(teamName, memberName, intent)) { + continue; + } + const route = routes[intent.id]; + if ( + intent.status === 'pending' && + !this.isCurrentPendingReportRoute(teamName, route, intent) + ) { return true; } } } for (const intent of Object.values((await this.readLegacyPendingFile(teamName)).intents)) { - if (intent.status === 'pending' && !indexedIds.has(intent.id)) { + if (!isReportIntentOwnedBy(teamName, intent.memberName, intent)) { + continue; + } + const route = routes[intent.id]; + if ( + intent.status === 'pending' && + !this.isCurrentPendingReportRoute(teamName, route, intent) + ) { return true; } } return false; } + private isCurrentPendingReportRoute( + teamName: string, + route: PendingReportsIndexFile['items'][string] | undefined, + intent: MemberWorkSyncReportIntent + ): boolean { + return ( + !!route && + normalizeTeamKey(intent.teamName) === normalizeTeamKey(teamName) && + route.status === 'pending' && + normalizeMemberKey(route.memberName) === normalizeMemberKey(intent.memberName) && + route.memberKey === this.paths.getMemberKey(intent.memberName) + ); + } + private async scanMemberOutboxes( teamName: string ): Promise<{ memberName: string; outbox: MemberOutboxFile }[]> { @@ -1412,27 +1604,56 @@ export class JsonMemberWorkSyncStore return outboxes; } - private async hasMissingIndexedDueOutboxItem( + private async hasUnindexedOrStaleIndexedDueOutboxItem( teamName: string, index: OutboxIndexFile, nowIso: string ): Promise { - const indexedIds = new Set(Object.keys(index.items)); - for (const { outbox } of await this.scanMemberOutboxes(teamName)) { + const routes = index.items; + for (const { memberName, outbox } of await this.scanMemberOutboxes(teamName)) { for (const item of Object.values(outbox.items)) { - if (canClaimOutboxItem(item, nowIso) && !indexedIds.has(item.id)) { + if (!isOutboxItemOwnedBy(teamName, memberName, item)) { + continue; + } + const route = routes[item.id]; + if ( + canClaimOutboxItem(item, nowIso) && + !this.isCurrentDueOutboxRoute(teamName, route, item, nowIso) + ) { return true; } } } for (const item of Object.values((await this.readLegacyOutboxFile(teamName)).items)) { - if (canClaimOutboxItem(item, nowIso) && !indexedIds.has(item.id)) { + if (!isOutboxItemOwnedBy(teamName, item.memberName, item)) { + continue; + } + const route = routes[item.id]; + if ( + canClaimOutboxItem(item, nowIso) && + !this.isCurrentDueOutboxRoute(teamName, route, item, nowIso) + ) { return true; } } return false; } + private isCurrentDueOutboxRoute( + teamName: string, + route: OutboxIndexRoute | undefined, + item: MemberWorkSyncOutboxItem, + nowIso: string + ): boolean { + return ( + !!route && + normalizeTeamKey(item.teamName) === normalizeTeamKey(teamName) && + normalizeMemberKey(route.memberName) === normalizeMemberKey(item.memberName) && + route.memberKey === this.paths.getMemberKey(item.memberName) && + canClaimOutboxRoute(route, nowIso) + ); + } + private async appendAudit(input: { teamName: string; memberName: string; diff --git a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts index 7365c92f..705bfe0b 100644 --- a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts +++ b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue.ts @@ -61,6 +61,7 @@ interface QueueItem { maxRunAt: number; triggerReasons: Set; triggerReasonCounts: Map; + retryCount: number; recovery?: MemberWorkSyncReconcileContext['recovery']; } @@ -84,9 +85,13 @@ export interface MemberWorkSyncEventQueueDeps { context: MemberWorkSyncReconcileContext ): Promise; isTeamActive(teamName: string): Promise | boolean; + reconcileInactiveTeams?: boolean; quietWindowMs?: number; triggerTiming?: Partial>>; concurrency?: number; + retryDelayMs?: number; + reconcileTimeoutMs?: number; + maxRetryAttempts?: number; now?: () => number; nowIso?: () => string; auditJournal?: MemberWorkSyncAuditJournalPort; @@ -101,12 +106,17 @@ function unrefTimer(timer: ReturnType): void { timer.unref?.(); } +const DEFAULT_RECONCILE_TIMEOUT_MS = 2 * 60_000; + export class MemberWorkSyncEventQueue { private readonly items = new Map(); private readonly running = new Map(); private readonly inFlight = new Set>(); private readonly quietWindowMs: number; private readonly concurrency: number; + private readonly retryDelayMs: number; + private readonly reconcileTimeoutMs: number; + private readonly maxRetryAttempts: number; private readonly now: () => number; private readonly nowIso: () => string; private timer: ReturnType | null = null; @@ -122,6 +132,9 @@ export class MemberWorkSyncEventQueue { constructor(private readonly deps: MemberWorkSyncEventQueueDeps) { this.quietWindowMs = deps.quietWindowMs ?? 90_000; this.concurrency = Math.max(1, deps.concurrency ?? 2); + this.retryDelayMs = Math.max(0, deps.retryDelayMs ?? 30_000); + this.reconcileTimeoutMs = Math.max(1, deps.reconcileTimeoutMs ?? DEFAULT_RECONCILE_TIMEOUT_MS); + this.maxRetryAttempts = Math.max(0, deps.maxRetryAttempts ?? 3); this.now = deps.now ?? Date.now; this.nowIso = deps.nowIso ?? (() => new Date().toISOString()); } @@ -209,6 +222,7 @@ export class MemberWorkSyncEventQueue { ? Math.min(existing.runAt, runAt) : Math.min(Math.max(existing.runAt, runAt), existing.maxRunAt); incrementReasonCount(existing.triggerReasonCounts, input.triggerReason); + existing.retryCount = 0; this.counters.coalesced += 1; this.appendAudit({ teamName, @@ -230,6 +244,7 @@ export class MemberWorkSyncEventQueue { maxRunAt: now + timing.maxCoalesceWaitMs, triggerReasons: new Set([input.triggerReason]), triggerReasonCounts: new Map([[input.triggerReason, 1]]), + retryCount: 0, ...(input.recovery ? { recovery: input.recovery } : {}), }); this.counters.enqueued += 1; @@ -366,8 +381,10 @@ export class MemberWorkSyncEventQueue { }; this.running.set(key, running); + let failed = false; const promise = this.executeItem(key, item, running) .catch((error: unknown) => { + failed = true; this.counters.failed += 1; this.deps.logger?.warn('member work sync queue reconcile failed', { teamName: item.teamName, @@ -380,6 +397,8 @@ export class MemberWorkSyncEventQueue { this.inFlight.delete(promise); if (running.rerunRequested && !this.stopped) { this.enqueueFollowUp(item, running); + } else if (failed && !this.stopped) { + this.enqueueRetryAfterFailure(key, item, running); } this.pump(); }); @@ -387,6 +406,53 @@ export class MemberWorkSyncEventQueue { this.inFlight.add(promise); } + private enqueueRetryAfterFailure(key: string, item: QueueItem, running: RunningItem): void { + if (item.retryCount >= this.maxRetryAttempts) { + this.counters.dropped += 1; + this.appendAudit({ + teamName: item.teamName, + memberName: item.memberName, + event: 'queue_dropped', + source: 'event_queue', + reason: 'reconcile_failed_max_retries', + triggerReasons: [...running.triggerReasons].sort(), + metadata: { + retryCount: item.retryCount, + maxRetryAttempts: this.maxRetryAttempts, + }, + }); + return; + } + + const now = this.now(); + const retryCount = item.retryCount + 1; + const recovery = running.recovery ?? item.recovery; + this.items.set(key, { + ...item, + lastQueuedAt: now, + runAt: now + this.retryDelayMs, + maxRunAt: now + this.retryDelayMs, + triggerReasons: new Set(running.triggerReasons), + triggerReasonCounts: new Map(item.triggerReasonCounts), + retryCount, + ...(recovery ? { recovery } : {}), + }); + this.appendAudit({ + teamName: item.teamName, + memberName: item.memberName, + event: 'queue_retry_scheduled', + source: 'event_queue', + reason: 'reconcile_failed', + triggerReasons: [...running.triggerReasons].sort(), + metadata: { + retryCount, + retryDelayMs: this.retryDelayMs, + maxRetryAttempts: this.maxRetryAttempts, + }, + }); + this.schedule(); + } + private enqueueFollowUp(item: QueueItem, running: RunningItem): void { const reasons = [...running.triggerReasons].sort(); const recovery = running.recovery ?? item.recovery; @@ -415,7 +481,7 @@ export class MemberWorkSyncEventQueue { } private async executeItem(_key: string, item: QueueItem, running: RunningItem): Promise { - if (!(await this.deps.isTeamActive(item.teamName))) { + if (!this.deps.reconcileInactiveTeams && !(await this.deps.isTeamActive(item.teamName))) { this.counters.dropped += 1; this.appendAudit({ teamName: item.teamName, @@ -428,7 +494,7 @@ export class MemberWorkSyncEventQueue { } const recovery = running.recovery ?? item.recovery; - await this.deps.reconcile( + await this.runReconcileWithTimeout( { teamName: item.teamName, memberName: item.memberName }, { reconciledBy: 'queue', @@ -446,6 +512,39 @@ export class MemberWorkSyncEventQueue { }); } + private async runReconcileWithTimeout( + input: { teamName: string; memberName: string }, + context: MemberWorkSyncReconcileContext + ): Promise { + let timeout: ReturnType | null = null; + let timedOut = false; + const reconcilePromise = this.deps.reconcile(input, { + ...context, + isCancelled: () => timedOut || context.isCancelled?.() === true, + }); + void reconcilePromise.catch(() => undefined); + try { + await Promise.race([ + reconcilePromise, + new Promise((_, reject) => { + timeout = setTimeout(() => { + timedOut = true; + reject( + new Error( + `member work sync queue reconcile timed out after ${this.reconcileTimeoutMs}ms` + ) + ); + }, this.reconcileTimeoutMs); + unrefTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } + private appendAudit(input: Omit): void { if (!this.deps.auditJournal) { return; diff --git a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncNudgeDispatchScheduler.ts b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncNudgeDispatchScheduler.ts index 276b04a1..aacf3f7b 100644 --- a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncNudgeDispatchScheduler.ts +++ b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncNudgeDispatchScheduler.ts @@ -4,6 +4,7 @@ import type { } from '../../core/application'; const DEFAULT_NUDGE_DISPATCH_INTERVAL_MS = 60_000; +const DEFAULT_NUDGE_DISPATCH_TIMEOUT_MS = 2 * 60_000; function uniqueNonEmpty(values: string[]): string[] { return [...new Set(values.map((value) => value.trim()).filter(Boolean))]; @@ -17,17 +18,23 @@ export interface MemberWorkSyncNudgeDispatchSchedulerDeps { listLifecycleActiveTeamNames(): Promise; dispatchDue(teamNames: string[]): Promise; intervalMs?: number; + dispatchTimeoutMs?: number; logger?: MemberWorkSyncLoggerPort; } export class MemberWorkSyncNudgeDispatchScheduler { private readonly intervalMs: number; + private readonly dispatchTimeoutMs: number; private timer: ReturnType | null = null; private running: Promise | null = null; private stopped = false; constructor(private readonly deps: MemberWorkSyncNudgeDispatchSchedulerDeps) { this.intervalMs = Math.max(10_000, deps.intervalMs ?? DEFAULT_NUDGE_DISPATCH_INTERVAL_MS); + this.dispatchTimeoutMs = Math.max( + 1, + deps.dispatchTimeoutMs ?? DEFAULT_NUDGE_DISPATCH_TIMEOUT_MS + ); } start(): void { @@ -84,11 +91,11 @@ export class MemberWorkSyncNudgeDispatchScheduler { private async dispatchOnce(): Promise { try { - const teamNames = uniqueNonEmpty(await this.deps.listLifecycleActiveTeamNames()); + const teamNames = uniqueNonEmpty(await this.listLifecycleActiveTeamNamesWithTimeout()); if (teamNames.length === 0) { return; } - const summary = await this.deps.dispatchDue(teamNames); + const summary = await this.runDispatchDueWithTimeout(teamNames); if (summary.claimed > 0 || summary.delivered > 0 || summary.retryable > 0) { this.deps.logger?.debug('member work sync scheduled nudge dispatch completed', { teamCount: teamNames.length, @@ -101,4 +108,56 @@ export class MemberWorkSyncNudgeDispatchScheduler { }); } } + + private async runDispatchDueWithTimeout( + teamNames: string[] + ): Promise { + let timeout: ReturnType | null = null; + const work = this.deps.dispatchDue(teamNames); + void work.catch(() => undefined); + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error( + `member work sync scheduled nudge dispatch timed out after ${this.dispatchTimeoutMs}ms` + ) + ); + }, this.dispatchTimeoutMs); + unrefTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } + + private async listLifecycleActiveTeamNamesWithTimeout(): Promise { + let timeout: ReturnType | null = null; + const work = this.deps.listLifecycleActiveTeamNames(); + void work.catch(() => undefined); + try { + return await Promise.race([ + work, + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error( + `member work sync scheduled nudge team listing timed out after ${this.dispatchTimeoutMs}ms` + ) + ); + }, this.dispatchTimeoutMs); + unrefTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } } diff --git a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncToolActivityBusySignal.ts b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncToolActivityBusySignal.ts index af25b61b..7cb33637 100644 --- a/src/features/member-work-sync/main/infrastructure/MemberWorkSyncToolActivityBusySignal.ts +++ b/src/features/member-work-sync/main/infrastructure/MemberWorkSyncToolActivityBusySignal.ts @@ -30,6 +30,10 @@ function parseIsoMs(value: string | undefined, fallbackMs: number): number { return Number.isFinite(parsed) ? parsed : fallbackMs; } +function parseEventIsoMs(value: string | undefined, nowMs: number): number { + return Math.min(parseIsoMs(value, nowMs), nowMs); +} + function addMsIso(baseIso: string, ms: number): string { return new Date(Date.parse(baseIso) + ms).toISOString(); } @@ -136,7 +140,7 @@ export class MemberWorkSyncToolActivityBusySignal implements MemberWorkSyncBusyS return; } const state = this.getOrCreateState(teamName, memberName); - const startedAtMs = parseIsoMs(startedAt, Date.now()); + const startedAtMs = parseEventIsoMs(startedAt, Date.now()); state.activeToolStartedAtByToolId.set(normalizedToolUseId, new Date(startedAtMs).toISOString()); state.recentBusyUntilByToolId.delete(normalizedToolUseId); } @@ -151,7 +155,7 @@ export class MemberWorkSyncToolActivityBusySignal implements MemberWorkSyncBusyS if (!memberName.trim() || !normalizedToolUseId) { return; } - const finishedAtMs = parseIsoMs(finishedAt, Date.now()); + const finishedAtMs = parseEventIsoMs(finishedAt, Date.now()); const busyUntilIso = new Date(finishedAtMs + this.busyGraceMs).toISOString(); const state = this.getOrCreateState(teamName, memberName); state.activeToolStartedAtByToolId.delete(normalizedToolUseId); diff --git a/src/features/member-work-sync/main/infrastructure/RuntimeTurnSettledDrainScheduler.ts b/src/features/member-work-sync/main/infrastructure/RuntimeTurnSettledDrainScheduler.ts index 1e82dc55..29cc2513 100644 --- a/src/features/member-work-sync/main/infrastructure/RuntimeTurnSettledDrainScheduler.ts +++ b/src/features/member-work-sync/main/infrastructure/RuntimeTurnSettledDrainScheduler.ts @@ -6,6 +6,7 @@ import type { export interface RuntimeTurnSettledDrainSchedulerDeps { drain(): Promise; intervalMs?: number; + drainTimeoutMs?: number; logger?: MemberWorkSyncLoggerPort; } @@ -13,14 +14,21 @@ function unrefTimer(timer: ReturnType): void { timer.unref?.(); } +const DEFAULT_RUNTIME_TURN_SETTLED_DRAIN_TIMEOUT_MS = 2 * 60_000; + export class RuntimeTurnSettledDrainScheduler { private readonly intervalMs: number; + private readonly drainTimeoutMs: number; private timer: ReturnType | null = null; private running = false; private disposed = false; constructor(private readonly deps: RuntimeTurnSettledDrainSchedulerDeps) { this.intervalMs = Math.max(1_000, deps.intervalMs ?? 15_000); + this.drainTimeoutMs = Math.max( + 1, + deps.drainTimeoutMs ?? DEFAULT_RUNTIME_TURN_SETTLED_DRAIN_TIMEOUT_MS + ); } start(): void { @@ -37,7 +45,7 @@ export class RuntimeTurnSettledDrainScheduler { this.running = true; try { - return await this.deps.drain(); + return await this.runDrainWithTimeout(); } catch (error) { this.deps.logger?.warn('runtime turn settled scheduled drain failed', { error: String(error), @@ -66,4 +74,25 @@ export class RuntimeTurnSettledDrainScheduler { }, delayMs); unrefTimer(this.timer); } + + private async runDrainWithTimeout(): Promise { + let timeout: ReturnType | null = null; + try { + return await Promise.race([ + this.deps.drain(), + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new Error(`runtime turn settled drain timed out after ${this.drainTimeoutMs}ms`) + ); + }, this.drainTimeoutMs); + unrefTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } } diff --git a/src/main/index.ts b/src/main/index.ts index af6bb697..46d16e8d 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -40,7 +40,10 @@ import { import { buildMemberWorkSyncRuntimeTurnSettledEnvironment, createMemberWorkSyncFeature, + hasUncertainWorkSyncRuntimeActivity, hasWorkSyncActiveRuntime, + isRuntimeMemberActivityUncertainForWorkSync, + isRuntimeMemberActiveForWorkSync, type MemberWorkSyncFeatureFacade, registerMemberWorkSyncIpc, removeMemberWorkSyncIpc, @@ -1820,7 +1823,10 @@ async function initializeServices(): Promise { teammateToolTracker = new TeammateToolTracker( teamMemberLogsFinder, teamLogSourceTracker, - forwardTeamChange + (event) => { + forwardTeamChange(event); + memberWorkSyncFeature?.noteTeamChange(event); + } ); // Allow TeamProvisioningService to trigger team refresh events (e.g. live lead replies). const teamChangeEmitter = (event: TeamChangeEvent): void => { @@ -1878,41 +1884,140 @@ async function initializeServices(): Promise { }); runtimeProviderManagementFeature = createRuntimeProviderManagementFeature(); const memberWorkSyncLogger = createLogger('Feature:MemberWorkSync'); - const hasMemberWorkSyncRuntimeActivity = async (teamName: string): Promise => { + const getMemberWorkSyncRuntimeSnapshot = async (input: { + teamName: string; + memberName?: string; + }) => { + const timeoutMs = 15_000; + let timer: ReturnType | null = null; + const snapshot = teamProvisioningService.getTeamAgentRuntimeSnapshot(input.teamName); + void snapshot.catch(() => undefined); try { - const snapshot = await teamProvisioningService.getTeamAgentRuntimeSnapshot(teamName); - return hasWorkSyncActiveRuntime(snapshot); + return await Promise.race([ + snapshot, + new Promise((resolve) => { + timer = setTimeout(() => { + memberWorkSyncLogger.warn('member work sync runtime snapshot timed out', { + teamName: input.teamName, + ...(input.memberName ? { memberName: input.memberName } : {}), + timeoutMs, + }); + resolve(null); + }, timeoutMs); + timer.unref?.(); + }), + ]); + } finally { + if (timer) { + clearTimeout(timer); + } + } + }; + const getMemberWorkSyncRuntimeActivity = async (teamName: string): Promise => { + try { + const snapshot = await getMemberWorkSyncRuntimeSnapshot({ teamName }); + if (!snapshot) { + return null; + } + const active = hasWorkSyncActiveRuntime(snapshot); + if (!active && hasUncertainWorkSyncRuntimeActivity(snapshot)) { + return null; + } + return active; } catch (error) { memberWorkSyncLogger.warn('member work sync runtime activity check failed', { teamName, error: String(error), }); - return false; + return null; + } + }; + const getMemberWorkSyncMemberRuntimeActivity = async (input: { + teamName: string; + memberName: string; + }): Promise => { + try { + const snapshot = await getMemberWorkSyncRuntimeSnapshot(input); + if (!snapshot) { + return null; + } + const active = isRuntimeMemberActiveForWorkSync(snapshot, input.memberName); + if (!active && isRuntimeMemberActivityUncertainForWorkSync(snapshot, input.memberName)) { + return null; + } + return active; + } catch (error) { + memberWorkSyncLogger.warn('member work sync member runtime activity check failed', { + teamName: input.teamName, + memberName: input.memberName, + error: String(error), + }); + return null; } }; const isTeamActiveForMemberWorkSync = async (teamName: string): Promise => { - if ( + const runtimeActive = await getMemberWorkSyncRuntimeActivity(teamName); + if (runtimeActive != null) { + return runtimeActive; + } + return ( teamProvisioningService.isTeamAlive(teamName) || teamProvisioningService.hasProvisioningRun(teamName) - ) { - return true; - } - return hasMemberWorkSyncRuntimeActivity(teamName); + ); }; const canDispatchMemberWorkSyncNudges = async (teamName: string): Promise => { - if (teamProvisioningService.isTeamAlive(teamName)) { - return true; + const runtimeActive = await getMemberWorkSyncRuntimeActivity(teamName); + if (runtimeActive != null) { + return runtimeActive; } - return hasMemberWorkSyncRuntimeActivity(teamName); + return teamProvisioningService.isTeamAlive(teamName); + }; + const isMemberActiveForMemberWorkSync = async (input: { + teamName: string; + memberName: string; + }): Promise => { + const runtimeActive = await getMemberWorkSyncMemberRuntimeActivity(input); + if (runtimeActive != null) { + return runtimeActive; + } + return ( + teamProvisioningService.isTeamAlive(input.teamName) || + teamProvisioningService.hasProvisioningRun(input.teamName) + ); }; const listMemberWorkSyncLifecycleActiveTeamNames = async (): Promise => { + const teams = (await teamDataService.listTeams()).filter((team) => !team.deletedAt); + const activeChecks = await Promise.allSettled( + teams.map(async (team) => { + try { + return { + teamName: team.teamName, + active: await isTeamActiveForMemberWorkSync(team.teamName), + }; + } catch (error) { + memberWorkSyncLogger.warn('member work sync lifecycle team activity check failed', { + teamName: team.teamName, + error: String(error), + }); + return { + teamName: team.teamName, + active: + teamProvisioningService.isTeamAlive(team.teamName) || + teamProvisioningService.hasProvisioningRun(team.teamName), + }; + } + }) + ); const activeTeamNames: string[] = []; - for (const team of await teamDataService.listTeams()) { - if (team.deletedAt) { + for (const check of activeChecks) { + if (check.status === 'rejected') { + memberWorkSyncLogger.warn('member work sync lifecycle team activity check failed', { + error: String(check.reason), + }); continue; } - if (await isTeamActiveForMemberWorkSync(team.teamName)) { - activeTeamNames.push(team.teamName); + if (check.value.active) { + activeTeamNames.push(check.value.teamName); } } return activeTeamNames; @@ -1924,6 +2029,7 @@ async function initializeServices(): Promise { kanbanManager: new TeamKanbanManager(), membersMetaStore: new TeamMembersMetaStore(), isTeamActive: isTeamActiveForMemberWorkSync, + isMemberActive: isMemberActiveForMemberWorkSync, canDispatchNudges: canDispatchMemberWorkSyncNudges, listLifecycleActiveTeamNames: listMemberWorkSyncLifecycleActiveTeamNames, extraBusySignals: [ @@ -2146,17 +2252,8 @@ async function initializeServices(): Promise { return Number.isFinite(expiresAtMs) && expiresAtMs > Date.now(); }); scheduleStartupTask(() => { - void teamDataService - .listTeams() - .then(async (teams) => { - const lifecycleActiveTeamNames = teams - .filter( - (team) => - !team.deletedAt && - (teamProvisioningService.isTeamAlive(team.teamName) || - teamProvisioningService.hasProvisioningRun(team.teamName)) - ) - .map((team) => team.teamName); + void listMemberWorkSyncLifecycleActiveTeamNames() + .then(async (lifecycleActiveTeamNames) => { await memberWorkSyncFeature?.replayPendingReports(lifecycleActiveTeamNames); await memberWorkSyncFeature?.enqueueStartupScan(lifecycleActiveTeamNames); }) diff --git a/src/main/ipc/configValidation.ts b/src/main/ipc/configValidation.ts index 3a247573..3ee53a10 100644 --- a/src/main/ipc/configValidation.ts +++ b/src/main/ipc/configValidation.ts @@ -52,6 +52,7 @@ const VALID_SECTIONS = new Set([ 'ssh', ]); const MAX_SNOOZE_MINUTES = 24 * 60; +const CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH = 200; const FIRST_PARTY_ANTHROPIC_HOSTS = new Set(['api.anthropic.com', 'api-staging.anthropic.com']); function isPlainObject(value: unknown): value is Record { @@ -66,6 +67,16 @@ function isFiniteNumber(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value); } +function hasControlCharacter(value: string): boolean { + for (let index = 0; index < value.length; index += 1) { + const code = value.charCodeAt(index); + if (code <= 31 || code === 127) { + return true; + } + } + return false; +} + function validateAnthropicCompatibleBaseUrl(value: string): string | null { const trimmed = value.trim(); if (!trimmed) { @@ -90,6 +101,47 @@ function validateAnthropicCompatibleBaseUrl(value: string): string | null { return null; } +function validateCodexCustomProviderBaseUrl(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + + try { + const url = new URL(trimmed); + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return 'providerConnections.codex.customProvider.baseUrl must use http:// or https://'; + } + if (url.username || url.password) { + return 'providerConnections.codex.customProvider.baseUrl must not include credentials'; + } + if (url.search || url.hash) { + return 'providerConnections.codex.customProvider.baseUrl must not include query or fragment'; + } + } catch { + return 'providerConnections.codex.customProvider.baseUrl must be a valid URL'; + } + + return null; +} + +function validateCodexCustomProviderModel(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed) { + return null; + } + + if (trimmed.length > CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH) { + return `providerConnections.codex.customProvider.model must be ${CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH} characters or fewer`; + } + + if (hasControlCharacter(trimmed)) { + return 'providerConnections.codex.customProvider.model must not include control characters'; + } + + return null; +} + function isValidTrigger(trigger: unknown): trigger is NotificationTrigger { if (!isPlainObject(trigger)) { return false; @@ -652,6 +704,83 @@ function validateProviderConnectionsSection( continue; } + if (connectionKey === 'customProvider') { + if (!isPlainObject(connectionValue)) { + return { + valid: false, + error: 'providerConnections.codex.customProvider must be an object', + }; + } + + const customProvider: Partial = {}; + for (const [customKey, customValue] of Object.entries(connectionValue)) { + if (customKey !== 'enabled' && customKey !== 'baseUrl' && customKey !== 'model') { + return { + valid: false, + error: `providerConnections.codex.customProvider.${customKey} is not a valid setting`, + }; + } + + if (customKey === 'enabled') { + if (typeof customValue !== 'boolean') { + return { + valid: false, + error: 'providerConnections.codex.customProvider.enabled must be a boolean', + }; + } + customProvider.enabled = customValue; + continue; + } + + if (customKey === 'baseUrl') { + if (typeof customValue !== 'string') { + return { + valid: false, + error: 'providerConnections.codex.customProvider.baseUrl must be a string', + }; + } + + const error = validateCodexCustomProviderBaseUrl(customValue); + if (error) { + return { valid: false, error }; + } + customProvider.baseUrl = customValue.trim(); + continue; + } + + if (typeof customValue !== 'string') { + return { + valid: false, + error: 'providerConnections.codex.customProvider.model must be a string', + }; + } + + const error = validateCodexCustomProviderModel(customValue); + if (error) { + return { valid: false, error }; + } + customProvider.model = customValue.trim(); + } + + if (customProvider.enabled === true && !customProvider.baseUrl?.trim()) { + return { + valid: false, + error: 'providerConnections.codex.customProvider.baseUrl is required when enabled', + }; + } + + if (customProvider.enabled === true && !customProvider.model?.trim()) { + return { + valid: false, + error: 'providerConnections.codex.customProvider.model is required when enabled', + }; + } + + codexUpdate.customProvider = + customProvider as ProviderConnectionsConfig['codex']['customProvider']; + continue; + } + return { valid: false, error: `providerConnections.codex.${connectionKey} is not a valid setting`, diff --git a/src/main/services/infrastructure/ConfigManager.ts b/src/main/services/infrastructure/ConfigManager.ts index dba96c79..70237b8e 100644 --- a/src/main/services/infrastructure/ConfigManager.ts +++ b/src/main/services/infrastructure/ConfigManager.ts @@ -282,6 +282,12 @@ export interface AnthropicCompatibleEndpointConfig { baseUrl: string; } +export interface CodexCustomProviderConfig { + enabled: boolean; + baseUrl: string; + model: string; +} + export interface ProviderConnectionsConfig { anthropic: { authMode: ProviderConnectionAuthMode; @@ -290,6 +296,7 @@ export interface ProviderConnectionsConfig { }; codex: { preferredAuthMode: CodexAccountAuthMode; + customProvider: CodexCustomProviderConfig; }; } @@ -392,6 +399,11 @@ const DEFAULT_CONFIG: AppConfig = { }, codex: { preferredAuthMode: 'auto', + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, }, }, runtime: { @@ -455,7 +467,8 @@ function normalizeConfiguredClaudeRootPath(value: unknown): string | null { function normalizeCodexPreferredAuthMode( currentValue: unknown, - legacyValue?: unknown + legacyValue?: unknown, + fallback: CodexAccountAuthMode = DEFAULT_CONFIG.providerConnections.codex.preferredAuthMode ): CodexAccountAuthMode { const candidate = currentValue ?? legacyValue; @@ -467,7 +480,7 @@ function normalizeCodexPreferredAuthMode( return 'chatgpt'; } - return DEFAULT_CONFIG.providerConnections.codex.preferredAuthMode; + return fallback; } function normalizeAnthropicCompatibleEndpointConfig( @@ -486,6 +499,22 @@ function normalizeAnthropicCompatibleEndpointConfig( }; } +function normalizeCodexCustomProviderConfig( + value: unknown, + fallback: CodexCustomProviderConfig = DEFAULT_CONFIG.providerConnections.codex.customProvider +): CodexCustomProviderConfig { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + return { ...fallback }; + } + + const raw = value as Partial; + return { + enabled: typeof raw.enabled === 'boolean' ? raw.enabled : fallback.enabled, + baseUrl: typeof raw.baseUrl === 'string' ? raw.baseUrl.trim() : fallback.baseUrl, + model: typeof raw.model === 'string' ? raw.model.trim() : fallback.model, + }; +} + function shouldPersistNormalizedConfig(loaded: Partial, normalized: AppConfig): boolean { return JSON.stringify(loaded) !== JSON.stringify(normalized); } @@ -673,6 +702,9 @@ export class ConfigManager { loaded.providerConnections?.codex?.preferredAuthMode, (loaded.providerConnections?.codex as { authMode?: unknown } | undefined)?.authMode ), + customProvider: normalizeCodexCustomProviderConfig( + loaded.providerConnections?.codex?.customProvider + ), }, }, runtime: { @@ -789,11 +821,14 @@ export class ConfigManager { ), }, codex: { - ...this.config.providerConnections.codex, - ...(connectionUpdate.codex ?? {}), preferredAuthMode: normalizeCodexPreferredAuthMode( connectionUpdate.codex?.preferredAuthMode, - (connectionUpdate.codex as { authMode?: unknown } | undefined)?.authMode + (connectionUpdate.codex as { authMode?: unknown } | undefined)?.authMode, + this.config.providerConnections.codex.preferredAuthMode + ), + customProvider: normalizeCodexCustomProviderConfig( + connectionUpdate.codex?.customProvider, + this.config.providerConnections.codex.customProvider ), }, } as unknown as Partial; diff --git a/src/main/services/runtime/ProviderConnectionService.ts b/src/main/services/runtime/ProviderConnectionService.ts index 8cf34f46..51fda8db 100644 --- a/src/main/services/runtime/ProviderConnectionService.ts +++ b/src/main/services/runtime/ProviderConnectionService.ts @@ -12,7 +12,10 @@ import { import { ApiKeyService } from '../extensions/apikeys/ApiKeyService'; import { ConfigManager } from '../infrastructure/ConfigManager'; -import type { AnthropicCompatibleEndpointConfig } from '../infrastructure/ConfigManager'; +import type { + AnthropicCompatibleEndpointConfig, + CodexCustomProviderConfig, +} from '../infrastructure/ConfigManager'; import type { CodexAccountAuthMode, CodexAccountSnapshotDto, @@ -27,6 +30,7 @@ import type { CliProviderAuthMode, CliProviderConnectionInfo, CliProviderId, + CliProviderModelCatalog, CliProviderReasoningEffort, CliProviderStatus, } from '@shared/types'; @@ -84,6 +88,9 @@ const CODEX_NATIVE_API_KEY_ENV_VAR = 'CODEX_API_KEY'; const CODEX_CLI_PATH_ENV_VAR = 'CODEX_CLI_PATH'; const CODEX_HOME_ENV_VAR = 'CODEX_HOME'; const CODEX_FORCED_LOGIN_METHOD_ENV_VAR = 'CLAUDE_CODE_CODEX_FORCED_LOGIN_METHOD'; +const CODEX_CUSTOM_PROVIDER_ID = 'agent_teams_custom'; +const CODEX_CUSTOM_PROVIDER_NAME = 'Agent Teams Custom'; +const CODEX_CUSTOM_PROVIDER_SETTINGS_KEY = 'agent_teams_custom_provider'; const CODEX_NATIVE_BACKEND_ID = 'codex-native'; const CODEX_LOGIN_STATUS_TIMEOUT_MS = 5_000; const ANTHROPIC_API_KEY_VERIFY_TIMEOUT_MS = 10_000; @@ -276,15 +283,127 @@ function isCodexExecBinary(binaryPath?: string | null): boolean { ); } +function tomlString(value: string): string { + return JSON.stringify(value); +} + +function buildCodexCustomProviderConfigOverrides(config: CodexCustomProviderConfig): string[] { + return [ + `model_provider=${tomlString(CODEX_CUSTOM_PROVIDER_ID)}`, + `model_providers.${CODEX_CUSTOM_PROVIDER_ID}.name=${tomlString(CODEX_CUSTOM_PROVIDER_NAME)}`, + `model_providers.${CODEX_CUSTOM_PROVIDER_ID}.base_url=${tomlString(config.baseUrl.trim())}`, + `model_providers.${CODEX_CUSTOM_PROVIDER_ID}.wire_api="responses"`, + `model_providers.${CODEX_CUSTOM_PROVIDER_ID}.env_key=${tomlString(CODEX_NATIVE_API_KEY_ENV_VAR)}`, + ]; +} + +function buildCodexLaunchArgs( + binaryPath: string | null | undefined, + loginMethod: 'chatgpt' | 'api', + configOverrides: readonly string[] = [] +): string[] { + if (isCodexExecBinary(binaryPath)) { + return [ + '-c', + `forced_login_method="${loginMethod}"`, + ...configOverrides.flatMap((override) => ['-c', override]), + ]; + } + + const codexSettings: Record = { forced_login_method: loginMethod }; + if (configOverrides.length > 0) { + codexSettings[CODEX_CUSTOM_PROVIDER_SETTINGS_KEY] = { + config_overrides: [...configOverrides], + }; + } + + return ['--settings', JSON.stringify({ codex: codexSettings })]; +} + function buildCodexForcedLoginLaunchArgs( binaryPath: string | null | undefined, loginMethod: 'chatgpt' | 'api' ): string[] { - if (isCodexExecBinary(binaryPath)) { - return ['-c', `forced_login_method="${loginMethod}"`]; + return buildCodexLaunchArgs(binaryPath, loginMethod); +} + +function isCodexCustomProviderBaseUrlUsable(baseUrl: string): boolean { + const trimmed = baseUrl.trim(); + if (!trimmed) { + return false; } - return ['--settings', JSON.stringify({ codex: { forced_login_method: loginMethod } })]; + try { + const url = new URL(trimmed); + return ( + (url.protocol === 'http:' || url.protocol === 'https:') && + !url.username && + !url.password && + !url.search && + !url.hash + ); + } catch { + return false; + } +} + +function isCodexCustomProviderModelUsable(model: string): boolean { + const trimmed = model.trim(); + if (trimmed.length === 0 || trimmed.length > 200) { + return false; + } + + for (let index = 0; index < trimmed.length; index += 1) { + const code = trimmed.charCodeAt(index); + if (code <= 31 || code === 127) { + return false; + } + } + + return true; +} + +function createCodexCustomProviderCatalog( + config: CodexCustomProviderConfig +): CliProviderModelCatalog { + const model = config.model.trim(); + const now = new Date(); + const staleAt = new Date(now.getTime() + 10 * 60_000); + return { + schemaVersion: 1, + providerId: 'codex', + source: 'static-fallback', + status: 'ready', + fetchedAt: now.toISOString(), + staleAt: staleAt.toISOString(), + defaultModelId: model, + defaultLaunchModel: model, + models: [ + { + id: model, + launchModel: model, + displayName: model, + hidden: false, + supportedReasoningEfforts: ['low', 'medium', 'high'], + defaultReasoningEffort: 'medium', + supportsFastMode: false, + inputModalities: ['text'], + supportsPersonality: false, + isDefault: true, + upgrade: false, + source: 'static-fallback', + badgeLabel: 'custom', + statusMessage: `Custom endpoint: ${config.baseUrl.trim()}`, + }, + ], + diagnostics: { + configReadState: 'skipped', + appServerState: 'healthy', + message: + 'Using app-managed Codex custom provider profile. Runtime support is verified during launch or model probe.', + code: 'agent-teams-custom-provider', + }, + }; } function applyCodexRuntimeContextEnv( @@ -426,6 +545,66 @@ export class ProviderConnectionService { return null; } + private getRawCodexCustomProvider(): CodexCustomProviderConfig { + const config = this.configManager.getConfig().providerConnections.codex.customProvider; + return { + enabled: config.enabled === true, + baseUrl: config.baseUrl.trim(), + model: config.model.trim(), + }; + } + + private getConfiguredCodexCustomProviderIssue(): string | null { + const config = this.getRawCodexCustomProvider(); + if (config.enabled !== true) { + return null; + } + + if (this.getConfiguredAuthMode('codex') !== 'api_key') { + return 'Codex custom provider is enabled but inactive because Codex auth mode is not API key.'; + } + + if (!config.baseUrl) { + return 'Codex custom provider is enabled, but no base URL is configured.'; + } + + if (!isCodexCustomProviderBaseUrlUsable(config.baseUrl)) { + return 'Codex custom provider base URL must use http:// or https:// and must not include credentials, query, or fragment.'; + } + + if (!config.model) { + return 'Codex custom provider is enabled, but no model is configured.'; + } + + if (!isCodexCustomProviderModelUsable(config.model)) { + return 'Codex custom provider model must be 200 characters or fewer and must not include control characters.'; + } + + return null; + } + + private getConfiguredCodexCustomProvider(): CodexCustomProviderConfig | null { + const config = this.getRawCodexCustomProvider(); + if ( + config.enabled !== true || + this.getConfiguredAuthMode('codex') !== 'api_key' || + !isCodexCustomProviderBaseUrlUsable(config.baseUrl) || + !isCodexCustomProviderModelUsable(config.model) + ) { + return null; + } + + return { + enabled: true, + baseUrl: config.baseUrl, + model: config.model, + }; + } + + getConfiguredCodexCustomProviderModel(): string | null { + return this.getConfiguredCodexCustomProvider()?.model ?? null; + } + private getConfiguredAnthropicCompatibleEndpoint(): AnthropicCompatibleEndpointConfig | null { const endpoint = this.configManager.getConfig().providerConnections.anthropic.compatibleEndpoint; @@ -776,6 +955,14 @@ export class ProviderConnectionService { return null; } + const customProviderIssue = + this.getConfiguredAuthMode('codex') === 'api_key' + ? this.getConfiguredCodexCustomProviderIssue() + : null; + if (customProviderIssue) { + return customProviderIssue; + } + const snapshot = await this.getCodexLaunchSnapshot(env, { refreshRuntimeMissing: true, refreshBlockedLaunch: true, @@ -902,11 +1089,16 @@ export class ProviderConnectionService { }); if (readiness.effectiveAuthMode === 'chatgpt') { - return buildCodexForcedLoginLaunchArgs(binaryPath, 'chatgpt'); + return buildCodexLaunchArgs(binaryPath, 'chatgpt'); } if (readiness.effectiveAuthMode === 'api_key') { - return buildCodexForcedLoginLaunchArgs(binaryPath, 'api'); + const customProvider = this.getConfiguredCodexCustomProvider(); + return buildCodexLaunchArgs( + binaryPath, + 'api', + customProvider ? buildCodexCustomProviderConfigOverrides(customProvider) : [] + ); } return []; @@ -929,6 +1121,47 @@ export class ProviderConnectionService { return withConnection; } + const customProvider = this.getConfiguredCodexCustomProvider(); + if (customProvider) { + const catalog = createCodexCustomProviderCatalog(customProvider); + const model = catalog.defaultLaunchModel ?? customProvider.model; + const statusMessage = + withConnection.statusMessage ?? + (withConnection.connection?.apiKeyConfigured + ? 'Codex custom provider configured' + : 'Codex custom provider configured. API key is not set.'); + + return { + ...withConnection, + models: [model], + modelCatalog: catalog, + subscriptionRateLimits: null, + backend: withConnection.backend + ? { + ...withConnection.backend, + endpointLabel: customProvider.baseUrl, + } + : { + kind: CODEX_NATIVE_BACKEND_ID, + label: 'Codex native', + endpointLabel: customProvider.baseUrl, + }, + runtimeCapabilities: { + ...withConnection.runtimeCapabilities, + modelCatalog: { + dynamic: false, + source: catalog.source, + }, + reasoningEffort: { + supported: true, + values: ['low', 'medium', 'high'] satisfies CliProviderReasoningEffort[], + configPassthrough: true, + }, + }, + statusMessage, + }; + } + try { if ( options.hydrateModelCatalog === false && @@ -1140,6 +1373,14 @@ export class ProviderConnectionService { : (externalCredential?.label ?? null); const compatibleEndpoint = providerId === 'anthropic' ? await this.getAnthropicCompatibleEndpointConnectionInfo() : null; + const codexCustomProvider = + providerId === 'codex' + ? { + config: this.getRawCodexCustomProvider(), + issueMessage: this.getConfiguredCodexCustomProviderIssue(), + active: Boolean(this.getConfiguredCodexCustomProvider()), + } + : null; return { ...capabilities, @@ -1165,6 +1406,13 @@ export class ProviderConnectionService { launchAllowed: codexSnapshot.launchAllowed, launchIssueMessage: codexSnapshot.launchIssueMessage, launchReadinessState: codexSnapshot.launchReadinessState, + customProvider: { + enabled: codexCustomProvider?.config.enabled ?? false, + active: codexCustomProvider?.active ?? false, + baseUrl: codexCustomProvider?.config.baseUrl ?? '', + model: codexCustomProvider?.config.model ?? '', + issueMessage: codexCustomProvider?.issueMessage ?? null, + }, } : null, }; diff --git a/src/main/services/runtime/providerModelProbe.ts b/src/main/services/runtime/providerModelProbe.ts index f706539c..6f093ce5 100644 --- a/src/main/services/runtime/providerModelProbe.ts +++ b/src/main/services/runtime/providerModelProbe.ts @@ -102,7 +102,15 @@ export function getProviderModelProbeTimeoutMs( } } -export function getProviderPreflightModel(providerId: TeamProviderId | undefined): string { +export function getProviderPreflightModel( + providerId: TeamProviderId | undefined, + options: { modelOverride?: string | null } = {} +): string { + const modelOverride = options.modelOverride?.trim(); + if (modelOverride) { + return modelOverride; + } + switch (resolveProbeProviderId(providerId)) { case 'codex': return 'gpt-5.4-mini'; @@ -114,6 +122,9 @@ export function getProviderPreflightModel(providerId: TeamProviderId | undefined } } -export function buildProviderPreflightPingArgs(providerId: TeamProviderId | undefined): string[] { - return buildProviderModelProbeArgs(getProviderPreflightModel(providerId)); +export function buildProviderPreflightPingArgs( + providerId: TeamProviderId | undefined, + options: { modelOverride?: string | null } = {} +): string[] { + return buildProviderModelProbeArgs(getProviderPreflightModel(providerId, options)); } diff --git a/src/main/services/team/ProcessBootstrapTransportEvidence.ts b/src/main/services/team/ProcessBootstrapTransportEvidence.ts index 8b452e70..0581f96e 100644 --- a/src/main/services/team/ProcessBootstrapTransportEvidence.ts +++ b/src/main/services/team/ProcessBootstrapTransportEvidence.ts @@ -54,6 +54,7 @@ const TRANSPORT_STAGE_LABELS: Record = { process_spawned: 'process spawned', stdout_attached: 'stdout attached', cli_started: 'CLI started', + startup_checkpoint: 'startup checkpoint', runtime_ready: 'runtime ready', inbox_poller_ready: 'inbox poller ready', mailbox_bootstrap_written: 'bootstrap mailbox row written', diff --git a/src/main/services/team/TeamLaunchFailureArtifactPack.ts b/src/main/services/team/TeamLaunchFailureArtifactPack.ts index 52892155..4f447d58 100644 --- a/src/main/services/team/TeamLaunchFailureArtifactPack.ts +++ b/src/main/services/team/TeamLaunchFailureArtifactPack.ts @@ -226,6 +226,8 @@ const WORKSPACE_TRUST_FAILURE_PATTERN = const BOOTSTRAP_TRANSPORT_EVIDENCE_PATTERN = new RegExp( [ 'mailbox_bootstrap_written', + 'startup_checkpoint', + 'last runtime stage', 'bootstrap_prompt_observed', 'bootstrap_submit_attempted', 'bootstrap_submitted', @@ -341,13 +343,15 @@ export function extractLaunchBootstrapTransportBreadcrumb( ): LaunchBootstrapTransportBreadcrumb { const parts = collectLaunchFailureSearchParts(input); const combined = parts.join('\n'); - const lastStageMatches = [...combined.matchAll(/last transport stage:\s*([^;\n]+)/gi)]; + const lastStageMatches = [ + ...combined.matchAll(/last (?:transport|runtime) stage:\s*([^;\n]+)/gi), + ]; const retryableMatches = [ ...combined.matchAll(/bootstrap_submit_rejected[^\n]*(?:retryable[=:]\s*(true|false))/gi), ]; const evidence = firstEvidence( parts, - /bootstrap_submit_|mailbox_bootstrap_written|bootstrap_prompt_observed|bootstrap_submitted|last transport stage|no stdin data received|local prompt handler/i + /bootstrap_submit_|mailbox_bootstrap_written|startup_checkpoint|bootstrap_prompt_observed|bootstrap_submitted|last (?:transport|runtime) stage|no stdin data received|local prompt handler/i ).map(redactLaunchFailureArtifactText); const retryableRaw = retryableMatches.at(-1)?.[1]?.toLowerCase(); return { diff --git a/src/main/services/team/TeamProvisioningService.ts b/src/main/services/team/TeamProvisioningService.ts index 9d44b02c..856da524 100644 --- a/src/main/services/team/TeamProvisioningService.ts +++ b/src/main/services/team/TeamProvisioningService.ts @@ -1478,7 +1478,11 @@ function classifyDeterministicBootstrapFailure(reason: string): { } function getPreflightPingArgs(providerId: TeamProviderId | undefined): string[] { - return buildProviderPreflightPingArgs(providerId); + const codexCustomModel = + resolveTeamProviderId(providerId) === 'codex' + ? ProviderConnectionService.getInstance().getConfiguredCodexCustomProviderModel() + : null; + return buildProviderPreflightPingArgs(providerId, { modelOverride: codexCustomModel }); } function getPreflightTimeoutMs(providerId: TeamProviderId | undefined): number { @@ -3286,6 +3290,17 @@ interface OpenCodeMemberInboxDelivery { userVisibleImpact?: OpenCodeRuntimeDeliveryUserVisibleImpact; } +class InboxRelayInFlightTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = 'InboxRelayInFlightTimeoutError'; + } +} + +function isInboxRelayInFlightTimeoutError(error: unknown): error is InboxRelayInFlightTimeoutError { + return error instanceof InboxRelayInFlightTimeoutError; +} + type OpenCodeVisibleReplyCorrelation = NonNullable< OpenCodePromptDeliveryLedgerRecord['visibleReplyCorrelation'] >; @@ -3444,6 +3459,15 @@ function getOpenCodeInboxRelayPriority( return 0; } +function getMemberInboxRelayPriority( + message: Pick +): number { + if (message.messageKind === 'member_work_sync_nudge') { + return 30; + } + return 0; +} + function getLeadInboxRelayPriority(message: Pick): number { if (message.messageKind === 'member_work_sync_nudge') { return 30; @@ -3478,6 +3502,13 @@ function compareOpenCodeInboxRelayMessagesByPriority( return compareInboxRelayMessages(a, b, getOpenCodeInboxRelayPriority); } +function compareMemberInboxRelayMessagesByPriority( + a: Pick & { messageId: string }, + b: Pick & { messageId: string } +): number { + return compareInboxRelayMessages(a, b, getMemberInboxRelayPriority); +} + function compareLeadInboxRelayMessagesByPriority( a: Pick & { messageId: string }, b: Pick & { messageId: string } @@ -3527,6 +3558,7 @@ export class TeamProvisioningService { private static readonly RETAINED_PROVISIONING_PROGRESS_TTL_MS = 5 * 60_000; private static readonly OPENCODE_RUNTIME_DELIVERY_ADVISORY_EVENT_TTL_MS = 24 * 60 * 60_000; private static readonly OPENCODE_RUNTIME_DELIVERY_LEAD_NOTICE_TTL_MS = 24 * 60 * 60_000; + private static readonly INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS = 2 * 60_000; private readonly runs = new Map(); private readonly provisioningRunByTeam = new Map(); @@ -6162,6 +6194,27 @@ export class TeamProvisioningService { return enabled; } + private async markOpenCodePromptLedgerFailedTerminal(input: { + ledger: OpenCodePromptDeliveryLedgerStore; + id: string; + reason: string; + diagnostics?: string[]; + failedAt: string; + eventContext?: Record; + }): Promise { + const failed = await input.ledger.markFailedTerminal({ + id: input.id, + reason: input.reason, + ...(input.diagnostics ? { diagnostics: input.diagnostics } : {}), + failedAt: input.failedAt, + }); + this.logOpenCodePromptDeliveryEvent('opencode_prompt_delivery_terminal_failure', failed, { + reason: input.reason, + ...(input.eventContext ?? {}), + }); + return failed; + } + private async findOpenCodeVisibleReplyByRelayOfMessageId(input: { teamName: string; replyRecipient?: string | null; @@ -7243,7 +7296,8 @@ export class TeamProvisioningService { input.ledgerRecord.maxSessionRefreshAttempts ?? OPENCODE_PROMPT_DELIVERY_SESSION_REFRESH_MAX_ATTEMPTS; if ((input.ledgerRecord.sessionRefreshAttempts ?? 0) >= maxSessionRefreshAttempts) { - return await input.ledger.markFailedTerminal({ + return await this.markOpenCodePromptLedgerFailedTerminal({ + ledger: input.ledger, id: input.ledgerRecord.id, reason: 'opencode_session_stale_observe_loop_after_accepted_prompt', diagnostics: [ @@ -7251,6 +7305,11 @@ export class TeamProvisioningService { `OpenCode session stayed stale while observing an accepted prompt after ${maxSessionRefreshAttempts} attempt(s).`, ], failedAt: now, + eventContext: { + observeOnlyAfterAcceptedPrompt: true, + sessionRefreshAttempts: input.ledgerRecord.sessionRefreshAttempts ?? 0, + maxSessionRefreshAttempts, + }, }); } const delayMs = OPENCODE_PROMPT_DELIVERY_RETRY_DELAY_MS; @@ -7287,7 +7346,8 @@ export class TeamProvisioningService { input.ledgerRecord.maxSessionRefreshAttempts ?? OPENCODE_PROMPT_DELIVERY_SESSION_REFRESH_MAX_ATTEMPTS; if ((input.ledgerRecord.sessionRefreshAttempts ?? 0) >= maxSessionRefreshAttempts) { - return await input.ledger.markFailedTerminal({ + return await this.markOpenCodePromptLedgerFailedTerminal({ + ledger: input.ledger, id: input.ledgerRecord.id, reason: 'opencode_session_refresh_loop_after_resolved_behavior_changed', diagnostics: [ @@ -7295,6 +7355,11 @@ export class TeamProvisioningService { `OpenCode session stayed stale after ${maxSessionRefreshAttempts} session refresh attempt(s).`, ], failedAt: now, + eventContext: { + retry: true, + sessionRefreshAttempts: input.ledgerRecord.sessionRefreshAttempts ?? 0, + maxSessionRefreshAttempts, + }, }); } const delayMs = this.getOpenCodeDeliveryNextDelayMs({ @@ -7338,10 +7403,12 @@ export class TeamProvisioningService { input.ledgerRecord.attempts >= input.ledgerRecord.maxAttempts && !canScheduleNoAssistantRecoveryRetry ) { - return await input.ledger.markFailedTerminal({ + return await this.markOpenCodePromptLedgerFailedTerminal({ + ledger: input.ledger, id: input.ledgerRecord.id, reason: input.reason, failedAt: now, + eventContext: { retry: input.retry }, }); } const delayMs = this.getOpenCodeDeliveryNextDelayMs({ @@ -11626,6 +11693,33 @@ export class TeamProvisioningService { return `opencode:${this.getMemberRelayKey(teamName, memberName)}`; } + private async waitForInboxRelayInFlight(input: { + promise: Promise; + relayName: string; + relayKey: string; + }): Promise { + let timer: ReturnType | null = null; + try { + return await Promise.race([ + input.promise, + new Promise((_, reject) => { + timer = setTimeout(() => { + reject( + new InboxRelayInFlightTimeoutError( + `${input.relayName} timed out after ${TeamProvisioningService.INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS}ms: ${input.relayKey}` + ) + ); + }, TeamProvisioningService.INBOX_RELAY_IN_FLIGHT_TIMEOUT_MS); + timer.unref?.(); + }), + ]); + } finally { + if (timer) { + clearTimeout(timer); + } + } + } + private normalizeRelayCandidateText(text: string): string { return stripAgentBlocks(String(text)).trim().replace(/\r\n/g, '\n'); } @@ -23166,7 +23260,23 @@ export class TeamProvisioningService { const relayKey = this.getMemberRelayKey(teamName, memberName); const existing = this.memberInboxRelayInFlight.get(relayKey); if (existing) { - return existing; + try { + return await this.waitForInboxRelayInFlight({ + promise: existing, + relayName: 'member_inbox_relay', + relayKey, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + logger.warn(`[${teamName}] member_inbox_relay_timed_out: ${getErrorMessage(error)}`); + return 0; + } finally { + if (this.memberInboxRelayInFlight.get(relayKey) === existing) { + this.memberInboxRelayInFlight.delete(relayKey); + } + } } const work = (async (): Promise => { @@ -23239,7 +23349,9 @@ export class TeamProvisioningService { if (actionableUnread.length === 0) return 0; const MAX_RELAY = 10; - const batch = actionableUnread.slice(0, MAX_RELAY); + const batch = [...actionableUnread] + .sort(compareMemberInboxRelayMessagesByPriority) + .slice(0, MAX_RELAY); this.armSilentTeammateForward(run, memberName, 'member_inbox_relay'); const rememberedRelayIds = this.rememberPendingInboxRelayCandidates(run, memberName, batch); @@ -23320,7 +23432,17 @@ export class TeamProvisioningService { this.memberInboxRelayInFlight.set(relayKey, work); try { - return await work; + return await this.waitForInboxRelayInFlight({ + promise: work, + relayName: 'member_inbox_relay', + relayKey, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + logger.warn(`[${teamName}] member_inbox_relay_timed_out: ${getErrorMessage(error)}`); + return 0; } finally { if (this.memberInboxRelayInFlight.get(relayKey) === work) { this.memberInboxRelayInFlight.delete(relayKey); @@ -23486,13 +23608,66 @@ export class TeamProvisioningService { if (existing) { const onlyMessageId = options.onlyMessageId?.trim(); if (!onlyMessageId) { - return existing; + try { + return await this.waitForInboxRelayInFlight({ + promise: existing, + relayName: 'opencode_member_inbox_relay', + relayKey, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + const diagnostic = `opencode_member_inbox_relay_timed_out: ${getErrorMessage(error)}`; + logger.warn(`[${teamName}] ${diagnostic}`); + return { + relayed: 0, + attempted: 0, + delivered: 0, + failed: 1, + lastDelivery: { + delivered: false, + accepted: false, + responsePending: false, + reason: 'opencode_member_inbox_relay_timed_out', + diagnostics: [diagnostic], + }, + diagnostics: [diagnostic], + }; + } finally { + if (this.openCodeMemberInboxRelayInFlight.get(relayKey) === existing) { + this.openCodeMemberInboxRelayInFlight.delete(relayKey); + } + } } const inboxMessages = await this.inboxReader .getMessagesFor(teamName, memberName) .catch(() => []); const targetMessage = inboxMessages.find((message) => message.messageId === onlyMessageId); if (targetMessage?.read) { + if (targetMessage.messageKind === 'member_work_sync_nudge') { + this.scheduleOpenCodeMemberInboxDeliveryWake({ + teamName, + memberName, + messageId: onlyMessageId, + delayMs: 500, + }); + const diagnostic = `opencode_work_sync_read_commit_waiting_for_active_relay: ${onlyMessageId}`; + return { + relayed: 0, + attempted: 1, + delivered: 0, + failed: 0, + lastDelivery: { + delivered: true, + accepted: false, + responsePending: true, + reason: 'opencode_work_sync_read_commit_waiting_for_active_relay', + diagnostics: [diagnostic], + }, + diagnostics: [diagnostic], + }; + } return { relayed: 0, attempted: 1, @@ -23576,7 +23751,7 @@ export class TeamProvisioningService { const onlyMessageId = options.onlyMessageId?.trim(); if (onlyMessageId) { const targetMessage = inboxMessages.find((message) => message.messageId === onlyMessageId); - if (targetMessage?.read) { + if (targetMessage?.read && targetMessage.messageKind !== 'member_work_sync_nudge') { return { relayed: 0, attempted: 1, @@ -23603,8 +23778,13 @@ export class TeamProvisioningService { } const unread = inboxMessages .filter((message): message is InboxMessage & { messageId: string } => { - if (message.read) return false; if (onlyMessageId && message.messageId !== onlyMessageId) return false; + if ( + message.read && + (!onlyMessageId || message.messageKind !== 'member_work_sync_nudge') + ) { + return false; + } if (typeof message.text !== 'string' || message.text.trim().length === 0) return false; return this.hasStableMessageId(message); }) @@ -23813,17 +23993,14 @@ export class TeamProvisioningService { pendingRecord ); } - failedRecord = await promptLedger.markFailedTerminal({ + failedRecord = await this.markOpenCodePromptLedgerFailedTerminal({ + ledger: promptLedger, id: pendingRecord.id, reason: attachmentPayloads.reason, diagnostics: attachmentPayloads.diagnostics, failedAt: nowIso(), + eventContext: { attachmentPayloadUnavailable: true }, }); - this.logOpenCodePromptDeliveryEvent( - 'opencode_prompt_delivery_response_observed', - failedRecord, - { attachmentPayloadUnavailable: true } - ); } catch (error) { const diagnostic = `opencode_inbox_attachment_terminal_ledger_failed: ${getErrorMessage( error @@ -23952,7 +24129,31 @@ export class TeamProvisioningService { this.openCodeMemberInboxRelayInFlight.set(relayKey, work); try { - return await work; + return await this.waitForInboxRelayInFlight({ + promise: work, + relayName: 'opencode_member_inbox_relay', + relayKey, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + const diagnostic = `opencode_member_inbox_relay_timed_out: ${getErrorMessage(error)}`; + logger.warn(`[${teamName}] ${diagnostic}`); + return { + relayed: 0, + attempted: options.onlyMessageId ? 1 : 0, + delivered: 0, + failed: 1, + lastDelivery: { + delivered: false, + accepted: false, + responsePending: false, + reason: 'opencode_member_inbox_relay_timed_out', + diagnostics: [diagnostic], + }, + diagnostics: [diagnostic], + }; } finally { if (this.openCodeMemberInboxRelayInFlight.get(relayKey) === work) { this.openCodeMemberInboxRelayInFlight.delete(relayKey); @@ -24261,7 +24462,23 @@ export class TeamProvisioningService { async relayLeadInboxMessages(teamName: string): Promise { const existing = this.leadInboxRelayInFlight.get(teamName); if (existing) { - return existing; + try { + return await this.waitForInboxRelayInFlight({ + promise: existing, + relayName: 'lead_inbox_relay', + relayKey: teamName, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + logger.warn(`[${teamName}] lead_inbox_relay_timed_out: ${getErrorMessage(error)}`); + return 0; + } finally { + if (this.leadInboxRelayInFlight.get(teamName) === existing) { + this.leadInboxRelayInFlight.delete(teamName); + } + } } const work = (async (): Promise => { @@ -24817,7 +25034,17 @@ export class TeamProvisioningService { this.leadInboxRelayInFlight.set(teamName, work); try { - return await work; + return await this.waitForInboxRelayInFlight({ + promise: work, + relayName: 'lead_inbox_relay', + relayKey: teamName, + }); + } catch (error) { + if (!isInboxRelayInFlightTimeoutError(error)) { + throw error; + } + logger.warn(`[${teamName}] lead_inbox_relay_timed_out: ${getErrorMessage(error)}`); + return 0; } finally { if (this.leadInboxRelayInFlight.get(teamName) === work) { this.leadInboxRelayInFlight.delete(teamName); diff --git a/src/main/services/team/TeamReconcileDrainScheduler.ts b/src/main/services/team/TeamReconcileDrainScheduler.ts index 0861ae10..4564b873 100644 --- a/src/main/services/team/TeamReconcileDrainScheduler.ts +++ b/src/main/services/team/TeamReconcileDrainScheduler.ts @@ -11,15 +11,61 @@ interface TeamReconcileDrainState { lastTrigger: TeamReconcileTrigger | null; } +const DEFAULT_TEAM_RECONCILE_DRAIN_RUN_TIMEOUT_MS = 2 * 60_000; + export interface TeamReconcileDrainScheduler { schedule(teamName: string, trigger: TeamReconcileTrigger): void; dispose(): void; } +class TeamReconcileDrainTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = 'TeamReconcileDrainTimeoutError'; + } +} + +function unrefTimer(timer: ReturnType): void { + timer.unref?.(); +} + +async function runWithTimeout(options: { + run: () => Promise; + timeoutMs: number; + teamName: string; + trigger: TeamReconcileTrigger; +}): Promise { + let timeout: ReturnType | null = null; + try { + await Promise.race([ + options.run(), + new Promise((_, reject) => { + timeout = setTimeout(() => { + reject( + new TeamReconcileDrainTimeoutError( + `team reconcile drain timed out for ${options.teamName} source=${options.trigger.source} detail=${options.trigger.detail} after ${options.timeoutMs}ms` + ) + ); + }, options.timeoutMs); + unrefTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } +} + export function createTeamReconcileDrainScheduler(options: { run: (teamName: string, trigger: TeamReconcileTrigger) => Promise; + runTimeoutMs?: number; }): TeamReconcileDrainScheduler { const states = new Map(); + const runTimeoutMs = Math.max( + 1, + options.runTimeoutMs ?? DEFAULT_TEAM_RECONCILE_DRAIN_RUN_TIMEOUT_MS + ); let disposed = false; const drainTeam = async (teamName: string): Promise => { @@ -40,9 +86,18 @@ export function createTeamReconcileDrainScheduler(options: { } try { - await options.run(teamName, trigger); + await runWithTimeout({ + run: () => options.run(teamName, trigger), + timeoutMs: runTimeoutMs, + teamName, + trigger, + }); } catch (error) { failed = true; + if (error instanceof TeamReconcileDrainTimeoutError && !state.pending) { + state.pending = true; + state.lastTrigger = trigger; + } throw error; } finally { if (!disposed) { @@ -54,10 +109,7 @@ export function createTeamReconcileDrainScheduler(options: { state.running = false; if (disposed || !state.pending) { states.delete(teamName); - return; - } - - if (failed) { + } else if (failed) { void drainTeam(teamName).catch(() => undefined); } } diff --git a/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts b/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts index be073191..8d99b5a6 100644 --- a/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts +++ b/src/main/services/team/provisioning/TeamProvisioningLaunchFailurePolicy.ts @@ -1,7 +1,4 @@ -import { - isProvisionedButNotAliveFailureReason, - stripProcessTableUnavailableDiagnosticSuffix, -} from '@shared/utils/teamLaunchFailureReason'; +import { stripProcessTableUnavailableDiagnosticSuffix } from '@shared/utils/teamLaunchFailureReason'; import { mentionsProcessTableUnavailable } from './TeamProvisioningLaunchDiagnostics'; import { isBootstrapInstructionPrompt } from './TeamProvisioningPromptBuilders'; @@ -72,7 +69,20 @@ export function isBootstrapMcpResourceReadFailureReason(reason?: string): boolea } export function isBootstrapCheckInTimeoutFailureReason(reason?: string): boolean { - return reason?.trim() === 'Teammate was registered but did not bootstrap-confirm before timeout.'; + const text = reason?.trim(); + if (!text) { + return false; + } + if (text === 'Teammate was registered but did not bootstrap-confirm before timeout.') { + return true; + } + const normalized = text.toLowerCase(); + return ( + normalized.includes('bootstrap prompt was submitted') && + normalized.includes('did not bootstrap-confirm') && + normalized.includes('submitted-confirmation timeout') && + normalized.includes('last transport stage: bootstrap_submitted') + ); } export function isBootstrapInstructionPromptFailureReason(reason?: string): boolean { diff --git a/src/main/services/team/stallMonitor/ActiveTeamRegistry.ts b/src/main/services/team/stallMonitor/ActiveTeamRegistry.ts index 75a54c77..597f6dd1 100644 --- a/src/main/services/team/stallMonitor/ActiveTeamRegistry.ts +++ b/src/main/services/team/stallMonitor/ActiveTeamRegistry.ts @@ -1,6 +1,10 @@ +import { createLogger } from '@shared/utils/logger'; + import type { TeamLogSourceTracker } from '../TeamLogSourceTracker'; import type { TeamChangeEvent } from '@shared/types'; +const logger = createLogger('Service:ActiveTeamRegistry'); + interface TeamAliveProcessesReader { listAliveProcessTeams(): Promise; } @@ -23,6 +27,8 @@ function unrefBackgroundTimer(timer: ReturnType): void { export class ActiveTeamRegistry { private readonly activeTeams = new Set(); + private readonly activationInFlight = new Set(); + private activationGeneration = 0; private reconcileTimer: ReturnType | null = null; constructor( @@ -41,8 +47,7 @@ export class ActiveTeamRegistry { (event.type === 'lead-activity' && event.detail !== 'offline') ) { if (!this.activeTeams.has(event.teamName)) { - this.activeTeams.add(event.teamName); - void this.teamLogSourceTracker.enableTracking(event.teamName, 'stall_monitor'); + void this.activateTeam(event.teamName); } return; } @@ -70,6 +75,7 @@ export class ActiveTeamRegistry { } async stop(): Promise { + this.activationGeneration += 1; if (this.reconcileTimer) { clearInterval(this.reconcileTimer); this.reconcileTimer = null; @@ -85,6 +91,7 @@ export class ActiveTeamRegistry { } async reconcile(): Promise { + const reconcileGeneration = this.activationGeneration; const aliveTeams = await this.teamDataService.listAliveProcessTeams(); const aliveSet = new Set(aliveTeams); @@ -92,8 +99,7 @@ export class ActiveTeamRegistry { if (this.activeTeams.has(teamName)) { continue; } - this.activeTeams.add(teamName); - await this.teamLogSourceTracker.enableTracking(teamName, 'stall_monitor'); + await this.activateTeam(teamName, reconcileGeneration); } for (const teamName of [...this.activeTeams]) { @@ -104,4 +110,41 @@ export class ActiveTeamRegistry { await this.teamLogSourceTracker.disableTracking(teamName, 'stall_monitor'); } } + + private async activateTeam( + teamName: string, + expectedGeneration = this.activationGeneration + ): Promise { + if (expectedGeneration !== this.activationGeneration) { + return; + } + if (this.activeTeams.has(teamName) || this.activationInFlight.has(teamName)) { + return; + } + + this.activationInFlight.add(teamName); + const activationGeneration = this.activationGeneration; + try { + await this.teamLogSourceTracker.enableTracking(teamName, 'stall_monitor'); + if (activationGeneration !== this.activationGeneration) { + await this.disableStaleActivation(teamName); + return; + } + this.activeTeams.add(teamName); + } catch (error) { + logger.warn(`Failed to enable stall-monitor tracking for ${teamName}: ${String(error)}`); + } finally { + this.activationInFlight.delete(teamName); + } + } + + private async disableStaleActivation(teamName: string): Promise { + try { + await this.teamLogSourceTracker.disableTracking(teamName, 'stall_monitor'); + } catch (error) { + logger.warn( + `Failed to disable stale stall-monitor tracking for ${teamName}: ${String(error)}` + ); + } + } } diff --git a/src/main/services/team/stallMonitor/TeamTaskStallJournal.ts b/src/main/services/team/stallMonitor/TeamTaskStallJournal.ts index b3d37792..52cd31b4 100644 --- a/src/main/services/team/stallMonitor/TeamTaskStallJournal.ts +++ b/src/main/services/team/stallMonitor/TeamTaskStallJournal.ts @@ -5,6 +5,8 @@ import * as path from 'path'; import { atomicWriteAsync } from '../atomicWrite'; import { withFileLock } from '../fileLock'; +import { getTeamTaskStallAlertCooldownMs } from './featureGates'; + import type { TaskStallEvaluation, TaskStallJournalEntry, @@ -15,7 +17,28 @@ function isValidState(value: unknown): value is TaskStallJournalState { return value === 'suspected' || value === 'alert_ready' || value === 'alerted'; } +function parseTime(value: string | undefined): number | null { + if (!value) { + return null; + } + const time = new Date(value).getTime(); + return Number.isFinite(time) ? time : null; +} + +export interface TeamTaskStallJournalOptions { + alertCooldownMs?: number; +} + export class TeamTaskStallJournal { + private readonly alertCooldownMs: number; + + constructor(options: TeamTaskStallJournalOptions = {}) { + this.alertCooldownMs = + options.alertCooldownMs != null && options.alertCooldownMs > 0 + ? options.alertCooldownMs + : getTeamTaskStallAlertCooldownMs(); + } + private getFilePath(teamName: string): string { return path.join(getTeamsBasePath(), teamName, 'stall-monitor-journal.json'); } @@ -67,6 +90,7 @@ export class TeamTaskStallJournal { epochKey, teamName: args.teamName, taskId: evaluation.taskId, + ...(evaluation.memberName ? { memberName: evaluation.memberName } : {}), branch: evaluation.branch, signal: evaluation.signal, state: 'suspected', @@ -78,7 +102,23 @@ export class TeamTaskStallJournal { } existing.updatedAt = args.now; + if (evaluation.memberName) { + existing.memberName = evaluation.memberName; + } if (existing.state === 'alerted') { + const nowMs = parseTime(args.now) ?? Date.now(); + const alertedAtMs = parseTime(existing.alertedAt); + if ( + alertedAtMs != null && + alertedAtMs <= nowMs && + nowMs - alertedAtMs < this.alertCooldownMs + ) { + continue; + } + + existing.state = 'alert_ready'; + existing.consecutiveScans += 1; + readyEvaluations.push(evaluation); continue; } @@ -138,6 +178,9 @@ export class TeamTaskStallJournal { ) .map((entry) => ({ ...entry, + ...(typeof entry.memberName === 'string' && entry.memberName.trim() + ? { memberName: entry.memberName } + : {}), ...(entry.alertedAt ? { alertedAt: entry.alertedAt } : {}), })); } catch (error) { diff --git a/src/main/services/team/stallMonitor/TeamTaskStallMonitor.ts b/src/main/services/team/stallMonitor/TeamTaskStallMonitor.ts index 02b197ac..7da2a4f5 100644 --- a/src/main/services/team/stallMonitor/TeamTaskStallMonitor.ts +++ b/src/main/services/team/stallMonitor/TeamTaskStallMonitor.ts @@ -26,6 +26,16 @@ interface TeamObservationState { lastActivationAtMs: number; } +interface TeamTaskStallMonitorOptions { + scanTimeoutMs?: number; +} + +interface TeamTaskStallScanRun { + cancelled: boolean; +} + +const DEFAULT_TEAM_TASK_STALL_SCAN_TIMEOUT_MS = 2 * 60_000; + function unrefBackgroundTimer(timer: ReturnType): void { const maybeTimer = timer as { unref?: () => void }; maybeTimer.unref?.(); @@ -37,14 +47,21 @@ export class TeamTaskStallMonitor { private scanInFlight = false; private started = false; private readonly observationByTeam = new Map(); + private readonly scanTimeoutMs: number; constructor( private readonly registry: ActiveTeamRegistry, private readonly snapshotSource: TeamTaskStallSnapshotSource, private readonly policy: TeamTaskStallPolicy, private readonly journal: TeamTaskStallJournal, - private readonly notifier: TeamTaskStallNotifier - ) {} + private readonly notifier: TeamTaskStallNotifier, + options: TeamTaskStallMonitorOptions = {} + ) { + this.scanTimeoutMs = Math.max( + 1, + options.scanTimeoutMs ?? DEFAULT_TEAM_TASK_STALL_SCAN_TIMEOUT_MS + ); + } start(): void { if (!isTeamTaskStallScannerEnabled()) { @@ -127,38 +144,87 @@ export class TeamTaskStallMonitor { return; } this.scanInFlight = true; + const scanRun: TeamTaskStallScanRun = { cancelled: false }; try { - const activeTeams = await this.registry.listActiveTeams(); - const activeSet = new Set(activeTeams); - for (const teamName of [...this.observationByTeam.keys()]) { - if (!activeSet.has(teamName)) { - this.observationByTeam.delete(teamName); - } - } - - const now = new Date(); - for (const teamName of activeTeams) { - const observation = this.getOrCreateObservation(teamName, now.getTime()); - const startupAgeMs = now.getTime() - observation.firstSeenAtMs; - if (startupAgeMs < getTeamTaskStallStartupGraceMs()) { - continue; - } - - const activationAgeMs = now.getTime() - observation.lastActivationAtMs; - if (activationAgeMs < getTeamTaskStallActivationGraceMs()) { - continue; - } - - await this.scanTeam(teamName, now); - } + await this.runScanWithTimeout(scanRun); } catch (error) { logger.warn(`Task stall monitor scan failed: ${String(error)}`); } finally { + scanRun.cancelled = true; this.scanInFlight = false; this.scheduleNextScan(getTeamTaskStallScanIntervalMs()); } } + private async runScanWithTimeout(scanRun: TeamTaskStallScanRun): Promise { + let timeout: ReturnType | null = null; + try { + await Promise.race([ + this.runScanBody(scanRun), + new Promise((_, reject) => { + timeout = setTimeout(() => { + scanRun.cancelled = true; + reject(new Error(`task stall monitor scan timed out after ${this.scanTimeoutMs}ms`)); + }, this.scanTimeoutMs); + unrefBackgroundTimer(timeout); + }), + ]); + } finally { + if (timeout) { + clearTimeout(timeout); + } + } + } + + private shouldContinueScan(scanRun: TeamTaskStallScanRun): boolean { + return this.started && !scanRun.cancelled; + } + + private async runScanBody(scanRun: TeamTaskStallScanRun): Promise { + const activeTeams = await this.registry.listActiveTeams(); + if (!this.shouldContinueScan(scanRun)) { + return; + } + const activeSet = new Set(activeTeams); + for (const teamName of [...this.observationByTeam.keys()]) { + if (!activeSet.has(teamName)) { + this.observationByTeam.delete(teamName); + } + } + + const now = new Date(); + const eligibleTeamNames: string[] = []; + for (const teamName of activeTeams) { + const observation = this.getOrCreateObservation(teamName, now.getTime()); + const startupAgeMs = now.getTime() - observation.firstSeenAtMs; + if (startupAgeMs < getTeamTaskStallStartupGraceMs()) { + continue; + } + + const activationAgeMs = now.getTime() - observation.lastActivationAtMs; + if (activationAgeMs < getTeamTaskStallActivationGraceMs()) { + continue; + } + + eligibleTeamNames.push(teamName); + } + + if (!this.shouldContinueScan(scanRun) || eligibleTeamNames.length === 0) { + return; + } + + const results = await Promise.allSettled( + eligibleTeamNames.map((teamName) => this.scanTeam(teamName, now, scanRun)) + ); + for (const [index, result] of results.entries()) { + if (result.status === 'rejected' && this.shouldContinueScan(scanRun)) { + logger.warn( + `Task stall monitor scan failed for ${eligibleTeamNames[index]}: ${String(result.reason)}` + ); + } + } + } + private getOrCreateObservation(teamName: string, nowMs: number): TeamObservationState { const existing = this.observationByTeam.get(teamName); if (existing) { @@ -172,8 +238,15 @@ export class TeamTaskStallMonitor { return created; } - private async scanTeam(teamName: string, now: Date): Promise { + private async scanTeam( + teamName: string, + now: Date, + scanRun: TeamTaskStallScanRun + ): Promise { const snapshot = await this.snapshotSource.getSnapshot(teamName); + if (!this.shouldContinueScan(scanRun)) { + return; + } if (!snapshot) { return; } @@ -203,6 +276,9 @@ export class TeamTaskStallMonitor { ...(scopedTaskIds ? { scopeTaskIds: scopedTaskIds } : {}), now: now.toISOString(), }); + if (!this.shouldContinueScan(scanRun)) { + return; + } const alerts = readyEvaluations .map((evaluation) => this.buildAlert(snapshot, evaluation)) @@ -215,6 +291,9 @@ export class TeamTaskStallMonitor { const alertedEpochKeys = new Set(); if (openCodeRemediationEnabled) { const remediatedAlerts = await this.notifier.notifyOpenCodeOwners(teamName, alerts); + if (!this.shouldContinueScan(scanRun)) { + return; + } for (const alert of remediatedAlerts) { alertedEpochKeys.add(alert.epochKey); } @@ -223,6 +302,9 @@ export class TeamTaskStallMonitor { const leadFallbackAlerts = alerts.filter((alert) => !alertedEpochKeys.has(alert.epochKey)); if (leadFallbackAlerts.length > 0 && isTeamTaskStallAlertsEnabled()) { await this.notifier.notifyLead(teamName, leadFallbackAlerts); + if (!this.shouldContinueScan(scanRun)) { + return; + } for (const alert of leadFallbackAlerts) { alertedEpochKeys.add(alert.epochKey); } @@ -233,6 +315,9 @@ export class TeamTaskStallMonitor { return; } + if (!this.shouldContinueScan(scanRun)) { + return; + } await Promise.all( alerts .filter((alert) => alertedEpochKeys.has(alert.epochKey)) diff --git a/src/main/services/team/stallMonitor/TeamTaskStallPolicy.ts b/src/main/services/team/stallMonitor/TeamTaskStallPolicy.ts index 8120f87b..d7dffd0c 100644 --- a/src/main/services/team/stallMonitor/TeamTaskStallPolicy.ts +++ b/src/main/services/team/stallMonitor/TeamTaskStallPolicy.ts @@ -304,6 +304,7 @@ function buildOpenCodeNoProgressEpochKey(args: { function buildAlertEvaluation(args: { task: TeamTask; + memberName?: string; branch: TaskStallBranch; signal: TaskStallSignal; progressSignal?: TaskProgressSignal; @@ -313,6 +314,7 @@ function buildAlertEvaluation(args: { return { status: 'alert', taskId: args.task.id, + ...(args.memberName ? { memberName: args.memberName } : {}), branch: args.branch, signal: args.signal, ...(args.progressSignal ? { progressSignal: args.progressSignal } : {}), @@ -330,6 +332,7 @@ function buildOpenCodeNoProgressAlertEvaluation(args: { return { status: 'alert', taskId: args.task.id, + memberName: args.owner, branch: 'work', signal: 'mid_turn_after_touch', progressSignal: 'unknown', @@ -488,6 +491,7 @@ export class TeamTaskStallPolicy { return buildAlertEvaluation({ task, + memberName: task.owner, branch: 'work', signal, progressSignal: progressClassification.signal, @@ -595,6 +599,7 @@ export class TeamTaskStallPolicy { return buildAlertEvaluation({ task, + memberName: resolvedReviewer.reviewer, branch: 'review', signal, touch: reviewContext.lastMeaningfulTouch, diff --git a/src/main/services/team/stallMonitor/TeamTaskStallTypes.ts b/src/main/services/team/stallMonitor/TeamTaskStallTypes.ts index 30c68581..f55f4a1f 100644 --- a/src/main/services/team/stallMonitor/TeamTaskStallTypes.ts +++ b/src/main/services/team/stallMonitor/TeamTaskStallTypes.ts @@ -46,6 +46,7 @@ export interface ResolvedReviewer { export interface TaskStallEvaluation { status: TaskStallEvaluationStatus; taskId?: string; + memberName?: string; branch?: TaskStallBranch; signal?: TaskStallSignal; progressSignal?: TaskProgressSignal; @@ -135,6 +136,7 @@ export interface TaskStallJournalEntry { epochKey: string; teamName: string; taskId: string; + memberName?: string; branch: TaskStallBranch; signal: TaskStallSignal; state: TaskStallJournalState; diff --git a/src/main/services/team/stallMonitor/featureGates.ts b/src/main/services/team/stallMonitor/featureGates.ts index 90db02a7..2878231c 100644 --- a/src/main/services/team/stallMonitor/featureGates.ts +++ b/src/main/services/team/stallMonitor/featureGates.ts @@ -55,6 +55,10 @@ export function getTeamTaskStallActivationGraceMs(): number { return readInt(process.env.CLAUDE_TEAM_TASK_STALL_ACTIVATION_GRACE_MS, 60_000); } +export function getTeamTaskStallAlertCooldownMs(): number { + return readInt(process.env.CLAUDE_TEAM_TASK_STALL_ALERT_COOLDOWN_MS, 10 * 60_000); +} + export function getOpenCodeWeakStartStallThresholdMs(): number { // Shorter OpenCode threshold for "started work" comments that do not contain concrete progress. return readInt(process.env.CLAUDE_TEAM_OPENCODE_WEAK_START_STALL_THRESHOLD_MS, 100_000); diff --git a/src/renderer/components/runtime/ProviderRuntimeSettingsDialog.tsx b/src/renderer/components/runtime/ProviderRuntimeSettingsDialog.tsx index 9384abe4..898a08d1 100644 --- a/src/renderer/components/runtime/ProviderRuntimeSettingsDialog.tsx +++ b/src/renderer/components/runtime/ProviderRuntimeSettingsDialog.tsx @@ -27,6 +27,7 @@ import { CodexLoginUserCodeBadge, } from '@renderer/components/runtime/CodexLoginLinkCopyButton'; import { Button } from '@renderer/components/ui/button'; +import { Checkbox } from '@renderer/components/ui/checkbox'; import { Dialog, DialogContent, @@ -70,7 +71,14 @@ import type { CliProviderAuthMode, CliProviderId, CliProviderStatus } from '@sha import type { ApiKeyEntry } from '@shared/types/extensions'; type ApiKeyProviderId = 'anthropic' | 'codex' | 'gemini'; -type PendingConnectionAction = 'auto' | 'oauth' | 'chatgpt' | 'api_key' | 'compatible' | null; +type PendingConnectionAction = + | 'auto' + | 'oauth' + | 'chatgpt' + | 'api_key' + | 'compatible' + | 'codex-custom-provider' + | null; interface ConnectionMethodCardOption { readonly authMode: CliProviderAuthMode; @@ -163,6 +171,7 @@ const API_KEY_PROVIDER_TRANSLATION_KEYS = { const ANTHROPIC_COMPATIBLE_AUTH_TOKEN_ENV_VAR = 'ANTHROPIC_AUTH_TOKEN'; const ANTHROPIC_COMPATIBLE_AUTH_TOKEN_NAME = 'Anthropic-compatible Auth Token'; +const CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH = 200; const FIRST_PARTY_ANTHROPIC_HOSTS = new Set(['api.anthropic.com', 'api-staging.anthropic.com']); function isApiKeyProviderId(providerId: CliProviderId): providerId is ApiKeyProviderId { @@ -231,6 +240,50 @@ function validateAnthropicCompatibleBaseUrl( return null; } +function validateCodexCustomProviderBaseUrl(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed) { + return 'Base URL is required when custom endpoint is enabled.'; + } + + try { + const url = new URL(trimmed); + if (url.protocol !== 'http:' && url.protocol !== 'https:') { + return 'Base URL must use http:// or https://.'; + } + if (url.username || url.password) { + return 'Base URL must not include username or password.'; + } + if (url.search || url.hash) { + return 'Base URL must not include query string or fragment.'; + } + } catch { + return 'Base URL must be a valid URL.'; + } + + return null; +} + +function validateCodexCustomProviderModel(value: string): string | null { + const trimmed = value.trim(); + if (!trimmed) { + return 'Model id is required when custom endpoint is enabled.'; + } + + if (trimmed.length > CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH) { + return `Model id must be ${CODEX_CUSTOM_PROVIDER_MODEL_MAX_LENGTH} characters or fewer.`; + } + + for (let index = 0; index < trimmed.length; index += 1) { + const code = trimmed.charCodeAt(index); + if (code <= 31 || code === 127) { + return 'Model id must not include newlines or control characters.'; + } + } + + return null; +} + function getConnectionDescription( provider: CliProviderStatus, t: ReturnType['t'] @@ -808,6 +861,12 @@ export const ProviderRuntimeSettingsDialog = ({ const [compatibleTokenValue, setCompatibleTokenValue] = useState(''); const [compatibleEndpointError, setCompatibleEndpointError] = useState(null); const [compatibleEndpointStatus, setCompatibleEndpointStatus] = useState(null); + const [codexCustomProviderEnabled, setCodexCustomProviderEnabled] = useState(false); + const [codexCustomProviderBaseUrl, setCodexCustomProviderBaseUrl] = useState(''); + const [codexCustomProviderModel, setCodexCustomProviderModel] = useState(''); + const [codexCustomProviderApiKeyValue, setCodexCustomProviderApiKeyValue] = useState(''); + const [codexCustomProviderError, setCodexCustomProviderError] = useState(null); + const [codexCustomProviderStatus, setCodexCustomProviderStatus] = useState(null); const apiKeyInputRef = useRef(null); const apiKeys = useStore((s) => s.apiKeys); @@ -854,6 +913,12 @@ export const ProviderRuntimeSettingsDialog = ({ setCompatibleTokenValue(''); setCompatibleEndpointError(null); setCompatibleEndpointStatus(null); + setCodexCustomProviderEnabled(false); + setCodexCustomProviderBaseUrl(''); + setCodexCustomProviderModel(''); + setCodexCustomProviderApiKeyValue(''); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); }, [open]); useEffect(() => { @@ -861,6 +926,8 @@ export const ProviderRuntimeSettingsDialog = ({ setRuntimeError(null); setCompatibleEndpointError(null); setCompatibleEndpointStatus(null); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); }, [selectedProviderId]); useEffect(() => { @@ -892,6 +959,11 @@ export const ProviderRuntimeSettingsDialog = ({ enabled: false, baseUrl: '', }; + const codexCustomProviderConfig = appConfig?.providerConnections?.codex.customProvider ?? { + enabled: false, + baseUrl: '', + model: '', + }; const selectedCompatibleToken = findPreferredApiKeyEntry( apiKeys, ANTHROPIC_COMPATIBLE_AUTH_TOKEN_ENV_VAR @@ -939,6 +1011,32 @@ export const ProviderRuntimeSettingsDialog = ({ nextConnection.configuredAuthMode = appConfig?.providerConnections?.codex.preferredAuthMode ?? mergedStatusProvider.connection.configuredAuthMode; + if (nextConnection.codex) { + nextConnection.codex = { + ...nextConnection.codex, + preferredAuthMode: + appConfig?.providerConnections?.codex.preferredAuthMode ?? + nextConnection.codex.preferredAuthMode, + customProvider: { + ...(nextConnection.codex.customProvider ?? { + enabled: false, + active: false, + baseUrl: '', + model: '', + issueMessage: null, + }), + enabled: codexCustomProviderConfig.enabled, + active: + codexCustomProviderConfig.enabled && + (appConfig?.providerConnections?.codex.preferredAuthMode ?? + mergedStatusProvider.connection.configuredAuthMode) === 'api_key' && + validateCodexCustomProviderBaseUrl(codexCustomProviderConfig.baseUrl) === null && + validateCodexCustomProviderModel(codexCustomProviderConfig.model) === null, + baseUrl: codexCustomProviderConfig.baseUrl, + model: codexCustomProviderConfig.model, + }, + }; + } } if (statusApiKeyConfig) { @@ -965,6 +1063,9 @@ export const ProviderRuntimeSettingsDialog = ({ appConfig?.providerConnections?.anthropic.authMode, appConfig?.providerConnections?.codex.preferredAuthMode, codexAccount.snapshot, + codexCustomProviderConfig.baseUrl, + codexCustomProviderConfig.enabled, + codexCustomProviderConfig.model, selectedCompatibleToken, selectedApiKey, statusApiKeyConfig, @@ -983,6 +1084,25 @@ export const ProviderRuntimeSettingsDialog = ({ setCompatibleEndpointStatus(null); }, [anthropicCompatibleConfig.baseUrl, open, selectedProviderId]); + useEffect(() => { + if (!open || selectedProviderId !== 'codex') { + return; + } + + setCodexCustomProviderEnabled(codexCustomProviderConfig.enabled); + setCodexCustomProviderBaseUrl(codexCustomProviderConfig.baseUrl); + setCodexCustomProviderModel(codexCustomProviderConfig.model); + setCodexCustomProviderApiKeyValue(''); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + }, [ + codexCustomProviderConfig.baseUrl, + codexCustomProviderConfig.enabled, + codexCustomProviderConfig.model, + open, + selectedProviderId, + ]); + const selectedProviderLoading = selectedProvider ? providerStatusLoading[selectedProvider.providerId] === true : false; @@ -1136,6 +1256,28 @@ export const ProviderRuntimeSettingsDialog = ({ (anthropicCompatibleTokenConfigured ? t('providerRuntime.status.configured') : null); const anthropicCompatibleMissingToken = anthropicCompatibleEndpointEnabled && !anthropicCompatibleTokenConfigured; + const codexCustomProvider = + selectedProvider?.providerId === 'codex' + ? (selectedProvider.connection?.codex?.customProvider ?? null) + : null; + const codexCustomProviderPersistedEnabled = + codexCustomProvider?.enabled ?? codexCustomProviderConfig.enabled; + const codexCustomProviderActive = codexCustomProvider?.active === true; + const codexCustomProviderIssueMessage = codexCustomProvider?.issueMessage ?? null; + const codexCustomProviderApiKeyConfigured = Boolean( + selectedProvider?.providerId === 'codex' && + (selectedApiKey || selectedProvider.connection?.apiKeyConfigured) + ); + const codexCustomProviderApiKeyStatus = + selectedApiKey?.maskedValue ?? + (selectedProvider?.providerId === 'codex' + ? selectedProvider.connection?.apiKeySourceLabel + : null) ?? + (codexCustomProviderApiKeyConfigured ? t('providerRuntime.status.configured') : null); + const codexCustomProviderInactiveMessage = + codexCustomProviderPersistedEnabled && configuredAuthMode !== 'api_key' + ? 'Custom endpoint is saved but inactive because Codex is not in API key mode.' + : null; useEffect(() => { if (!showApiKeyForm) { @@ -1194,6 +1336,8 @@ export const ProviderRuntimeSettingsDialog = ({ return t('providerRuntime.progress.switchingApiKeyMode'); case 'auto': return t('providerRuntime.progress.switchingAuto'); + case 'codex-custom-provider': + return 'Saving Codex custom endpoint'; default: return t('providerRuntime.progress.applyingConnectionChanges'); } @@ -1443,6 +1587,146 @@ export const ProviderRuntimeSettingsDialog = ({ } }; + const handleSaveCodexCustomProvider = async (): Promise => { + if (selectedProvider?.providerId !== 'codex' || !apiKeyConfig) { + return; + } + + const baseUrl = codexCustomProviderBaseUrl.trim(); + const model = codexCustomProviderModel.trim(); + const shouldEnable = codexCustomProviderEnabled; + if (shouldEnable) { + const baseUrlError = validateCodexCustomProviderBaseUrl(baseUrl); + if (baseUrlError) { + setCodexCustomProviderError(baseUrlError); + setCodexCustomProviderStatus(null); + return; + } + + const modelError = validateCodexCustomProviderModel(model); + if (modelError) { + setCodexCustomProviderError(modelError); + setCodexCustomProviderStatus(null); + return; + } + } else if (baseUrl) { + const baseUrlError = validateCodexCustomProviderBaseUrl(baseUrl); + if (baseUrlError) { + setCodexCustomProviderError(baseUrlError); + setCodexCustomProviderStatus(null); + return; + } + } + + if (!shouldEnable && model) { + const modelError = validateCodexCustomProviderModel(model); + if (modelError) { + setCodexCustomProviderError(modelError); + setCodexCustomProviderStatus(null); + return; + } + } + + setConnectionSaving(true); + setPendingConnectionAction('codex-custom-provider'); + setConnectionError(null); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + let updateSucceeded = false; + + try { + if (codexCustomProviderApiKeyValue.trim()) { + await saveApiKey({ + id: selectedApiKey?.id, + name: apiKeyConfig.name, + envVarName: apiKeyConfig.envVarName, + value: codexCustomProviderApiKeyValue.trim(), + scope: selectedApiKey?.scope ?? 'user', + }); + } + + await updateConfig('providerConnections', { + codex: { + ...(shouldEnable ? { preferredAuthMode: 'api_key' as const } : {}), + customProvider: { + enabled: shouldEnable, + baseUrl, + model, + }, + }, + }); + updateSucceeded = true; + setCodexCustomProviderApiKeyValue(''); + setCodexCustomProviderStatus( + shouldEnable + ? 'Custom endpoint saved. Codex API key mode is selected.' + : 'Custom endpoint disabled. Saved endpoint, model, and key were kept.' + ); + } catch (error) { + setCodexCustomProviderError( + error instanceof Error ? error.message : 'Failed to save Codex custom endpoint.' + ); + } finally { + if (updateSucceeded) { + try { + await codexAccount.refresh({ includeRateLimits: true, forceRefreshToken: true }); + await onRefreshProvider?.('codex'); + } catch { + setConnectionError('Codex custom endpoint saved, but provider status refresh failed.'); + } + } + + setConnectionSaving(false); + setPendingConnectionAction(null); + } + }; + + const handleDisableCodexCustomProvider = async (): Promise => { + if (selectedProvider?.providerId !== 'codex') { + return; + } + + setConnectionSaving(true); + setPendingConnectionAction('codex-custom-provider'); + setConnectionError(null); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + let updateSucceeded = false; + + try { + await updateConfig('providerConnections', { + codex: { + customProvider: { + enabled: false, + baseUrl: codexCustomProviderConfig.baseUrl, + model: codexCustomProviderConfig.model, + }, + }, + }); + updateSucceeded = true; + setCodexCustomProviderEnabled(false); + setCodexCustomProviderStatus( + 'Custom endpoint disabled. Saved endpoint, model, and key were kept.' + ); + } catch (error) { + setCodexCustomProviderError( + error instanceof Error ? error.message : 'Failed to disable Codex custom endpoint.' + ); + } finally { + if (updateSucceeded) { + try { + await codexAccount.refresh({ includeRateLimits: true, forceRefreshToken: true }); + await onRefreshProvider?.('codex'); + } catch { + setConnectionError('Codex custom endpoint disabled, but provider status refresh failed.'); + } + } + + setConnectionSaving(false); + setPendingConnectionAction(null); + } + }; + const handleCodexAccountRefresh = async (): Promise => { setConnectionError(null); try { @@ -1891,6 +2175,255 @@ export const ProviderRuntimeSettingsDialog = ({ ) : null} + {selectedProvider.providerId === 'codex' ? ( +
+
+
+
+ Custom API endpoint +
+
+ Route Codex API-key launches through an app-managed custom provider. +
+
+
+ + {codexCustomProviderPersistedEnabled ? 'enabled' : 'off'} + + {codexCustomProviderPersistedEnabled ? ( + + {codexCustomProviderActive ? 'active' : 'inactive'} + + ) : null} +
+
+ +
+ { + setCodexCustomProviderEnabled(checked === true); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + }} + /> + + Enable custom endpoint for Codex API-key launches + +
+ +
+
+ + { + setCodexCustomProviderBaseUrl(event.currentTarget.value); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + }} + placeholder="https://gateway.example.com/v1" + className="h-9 text-sm" + disabled={connectionBusy} + /> +
+ +
+ + { + setCodexCustomProviderModel(event.currentTarget.value); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + }} + placeholder="gateway-model-id" + className="h-9 text-sm" + disabled={connectionBusy} + /> +
+
+ +
+ + { + setCodexCustomProviderApiKeyValue(event.currentTarget.value); + setCodexCustomProviderError(null); + setCodexCustomProviderStatus(null); + }} + placeholder={ + codexCustomProviderApiKeyConfigured + ? 'Keep saved OPENAI_API_KEY' + : apiKeyConfig?.placeholder + } + className="h-9 text-sm" + disabled={connectionBusy || apiKeySaving} + /> +
+ +
+ + API key:{' '} + {codexCustomProviderApiKeyConfigured + ? t('providerRuntime.status.configured') + : t('providerRuntime.status.notSet')} + + {codexCustomProviderApiKeyStatus ? ( + + {codexCustomProviderApiKeyStatus} + + ) : null} + {codexCustomProviderPersistedEnabled && codexCustomProvider?.baseUrl ? ( + + {codexCustomProvider.baseUrl} + + ) : null} +
+ +
+ + + Endpoint must support the Codex Responses API. Chat Completions-only + gateways may fail at launch or model probe time. + +
+ + {codexCustomProviderError ? ( +
+ + {codexCustomProviderError} +
+ ) : codexCustomProviderStatus ? ( +
+ {codexCustomProviderStatus} +
+ ) : codexCustomProviderIssueMessage || + codexCustomProviderInactiveMessage || + (codexCustomProviderPersistedEnabled && + !codexCustomProviderApiKeyConfigured) ? ( +
+ + + {codexCustomProviderIssueMessage ?? + codexCustomProviderInactiveMessage ?? + 'Custom endpoint is enabled, but no OPENAI_API_KEY is configured.'} + +
+ ) : null} + +
+ {codexCustomProviderPersistedEnabled ? ( + + ) : null} + +
+
+ ) : null} +
{configuredAuthMode && !hideConnectionMethodMeta ? ( { endpointLabel: 'codex exec --json', }); }); + + it('preserves an active Codex custom provider endpoint label through snapshot merge', () => { + const provider = createBaseCodexProvider(); + const customProvider = { + enabled: true, + active: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + issueMessage: null, + }; + + const merged = mergeCodexProviderStatusWithSnapshot( + { + ...provider, + backend: { + ...provider.backend!, + endpointLabel: customProvider.baseUrl, + }, + connection: { + ...provider.connection!, + configuredAuthMode: 'api_key', + codex: { + ...provider.connection!.codex!, + preferredAuthMode: 'api_key', + effectiveAuthMode: 'api_key', + customProvider, + }, + }, + }, + { + ...createReadyChatgptSnapshot(), + preferredAuthMode: 'api_key', + effectiveAuthMode: 'api_key', + launchReadinessState: 'ready_api_key', + managedAccount: null, + } + ); + + expect(merged.backend?.endpointLabel).toBe('https://gateway.example.com/v1'); + expect(merged.connection?.codex?.customProvider).toEqual(customProvider); + }); }); diff --git a/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts b/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts index 92b983a7..af3d67cf 100644 --- a/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts +++ b/test/features/member-work-sync/core/MemberWorkSyncUseCases.test.ts @@ -6,6 +6,7 @@ import { MemberWorkSyncNudgeDispatcher, type MemberWorkSyncOutboxStorePort, MemberWorkSyncPendingReportIntentReplayer, + MemberWorkSyncReconcileCancelledError, MemberWorkSyncReconciler, MemberWorkSyncReporter, type MemberWorkSyncReviewPickupDeliveryPort, @@ -13,7 +14,7 @@ import { type MemberWorkSyncStatusStorePort, type MemberWorkSyncUseCaseDeps, } from '@features/member-work-sync/core/application'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import type { MemberWorkSyncActionableWorkItem, @@ -79,6 +80,10 @@ const secondReviewPickupItem: MemberWorkSyncActionableWorkItem = { }, }; +function isTerminalOutboxStatus(status: MemberWorkSyncOutboxItem['status']): boolean { + return status === 'delivered' || status === 'superseded' || status === 'failed_terminal'; +} + class MutableClock { private current = new Date('2026-04-29T00:00:00.000Z'); @@ -231,7 +236,7 @@ class InMemoryOutboxStore implements MemberWorkSyncOutboxStorePort { async markDelivered(input: MemberWorkSyncOutboxMarkDeliveredInput): Promise { const current = this.items.get(input.id); - if (current?.attemptGeneration === input.attemptGeneration) { + if (current?.attemptGeneration === input.attemptGeneration && current.status === 'claimed') { const next = { ...current, status: 'delivered' as const, @@ -254,7 +259,10 @@ class InMemoryOutboxStore implements MemberWorkSyncOutboxStorePort { async markFailed(input: MemberWorkSyncOutboxMarkFailedInput): Promise { const current = this.items.get(input.id); - if (current?.attemptGeneration === input.attemptGeneration) { + if ( + current?.attemptGeneration === input.attemptGeneration && + !isTerminalOutboxStatus(current.status) + ) { this.items.set(input.id, { ...current, status: input.retryable ? 'failed_retryable' : 'failed_terminal', @@ -324,10 +332,12 @@ function createDeps(options?: { activeMemberNames?: string[]; inactive?: boolean; teamActive?: boolean; + memberActive?: boolean; providerId?: 'opencode' | 'codex'; outboxStore?: MemberWorkSyncOutboxStorePort; inboxNudge?: MemberWorkSyncInboxNudgePort; busySignal?: MemberWorkSyncUseCaseDeps['busySignal']; + watchdogCooldown?: MemberWorkSyncUseCaseDeps['watchdogCooldown']; reviewPickupDelivery?: MemberWorkSyncReviewPickupDeliveryPort; reviewPickupEscalation?: MemberWorkSyncReviewPickupEscalationPort; }) { @@ -361,6 +371,7 @@ function createDeps(options?: { ...(options?.outboxStore ? { outboxStore: options.outboxStore } : {}), ...(options?.inboxNudge ? { inboxNudge: options.inboxNudge } : {}), ...(options?.busySignal ? { busySignal: options.busySignal } : {}), + ...(options?.watchdogCooldown ? { watchdogCooldown: options.watchdogCooldown } : {}), ...(options?.reviewPickupDelivery ? { reviewPickupDelivery: options.reviewPickupDelivery } : {}), @@ -379,6 +390,7 @@ function createDeps(options?: { }, lifecycle: { isTeamActive: () => options?.teamActive ?? true, + isMemberActive: () => options?.memberActive ?? true, }, auditJournal: { append: async (event) => { @@ -414,6 +426,71 @@ describe('MemberWorkSync use cases', () => { ]); }); + it('does not write status or plan nudges after a queued reconcile is cancelled', async () => { + const outbox = new InMemoryOutboxStore(); + const { auditEvents, deps, store } = createDeps({ outboxStore: outbox }); + + await expect( + new MemberWorkSyncReconciler(deps).execute( + { teamName: 'team-a', memberName: 'bob' }, + { + reconciledBy: 'queue', + triggerReasons: ['turn_settled'], + isCancelled: () => true, + } + ) + ).rejects.toBeInstanceOf(MemberWorkSyncReconcileCancelledError); + + expect(store.writes).toHaveLength(0); + expect(outbox.ensures).toHaveLength(0); + expect(auditEvents.map((event) => event.event)).toEqual(['reconcile_started']); + }); + + it('does not create a report token when a queued reconcile is cancelled after decision audit', async () => { + const outbox = new InMemoryOutboxStore(); + const { auditEvents, deps, store } = createDeps({ outboxStore: outbox }); + let cancelled = false; + let tokenCreates = 0; + deps.auditJournal = { + append: async (event) => { + auditEvents.push(event); + if (event.event === 'decision_made') { + cancelled = true; + } + }, + }; + deps.reportToken = { + create: async (input) => { + tokenCreates += 1; + return { + token: `token:${input.teamName}:${input.memberName}:${input.agendaFingerprint}`, + expiresAt: '2026-04-29T00:15:00.000Z', + }; + }, + verify: async () => ({ ok: false, reason: 'missing' }), + }; + + await expect( + new MemberWorkSyncReconciler(deps).execute( + { teamName: 'team-a', memberName: 'bob' }, + { + reconciledBy: 'queue', + triggerReasons: ['turn_settled'], + isCancelled: () => cancelled, + } + ) + ).rejects.toBeInstanceOf(MemberWorkSyncReconcileCancelledError); + + expect(tokenCreates).toBe(0); + expect(store.writes).toHaveLength(0); + expect(outbox.ensures).toHaveLength(0); + expect(auditEvents.map((event) => event.event)).toEqual([ + 'reconcile_started', + 'agenda_loaded', + 'decision_made', + ]); + }); + it('accepts still_working as a bounded lease for the current fingerprint', async () => { const { auditEvents, clock, deps } = createDeps(); const reader = new MemberWorkSyncReconciler(deps); @@ -447,6 +524,36 @@ describe('MemberWorkSync use cases', () => { expect(auditEvents.map((event) => event.event)).toContain('report_accepted'); }); + it('rejects reports when this member runtime is no longer active', async () => { + const { deps } = createDeps(); + const reader = new MemberWorkSyncReconciler(deps); + const current = await reader.execute({ teamName: 'team-a', memberName: 'bob' }); + const reporter = new MemberWorkSyncReporter({ + ...deps, + lifecycle: { + isTeamActive: () => true, + isMemberActive: () => false, + }, + }); + + const result = await reporter.execute({ + teamName: 'team-a', + memberName: 'bob', + state: 'still_working', + agendaFingerprint: current.agenda.fingerprint, + reportToken: current.reportToken, + source: 'test', + }); + + expect(result.accepted).toBe(false); + expect(result.code).toBe('member_runtime_inactive'); + expect(result.status.state).toBe('inactive'); + expect(result.status.report).toMatchObject({ + accepted: false, + rejectionCode: 'member_runtime_inactive', + }); + }); + it('uses app clock instead of model supplied reportedAt for lease timing', async () => { const { deps } = createDeps(); const reader = new MemberWorkSyncReconciler(deps); @@ -577,6 +684,18 @@ describe('MemberWorkSync use cases', () => { expect(status.shadow?.wouldNudge).toBe(false); }); + it('marks status inactive when this member runtime is not active', async () => { + const { deps } = createDeps({ memberActive: false }); + const status = await new MemberWorkSyncReconciler(deps).execute({ + teamName: 'team-a', + memberName: 'bob', + }); + + expect(status.state).toBe('inactive'); + expect(status.diagnostics).toContain('member_runtime_inactive'); + expect(status.shadow?.wouldNudge).toBe(false); + }); + it('records fingerprint transitions without treating them as progress proof', async () => { const { deps, source } = createDeps(); const reader = new MemberWorkSyncReconciler(deps); @@ -892,6 +1011,379 @@ describe('MemberWorkSync use cases', () => { }); }); + it('supersedes due nudges for inactive member runtimes without inbox delivery', async () => { + const outbox = new InMemoryOutboxStore(); + const inbox = new InMemoryInboxNudge(); + const { deps, store } = createDeps({ outboxStore: outbox, inboxNudge: inbox }); + store.phase2ReadinessState = 'shadow_ready'; + + const status = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const dispatcher = new MemberWorkSyncNudgeDispatcher({ + ...deps, + lifecycle: { + isTeamActive: () => true, + isMemberActive: () => false, + }, + }); + + const summary = await dispatcher.dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(summary).toMatchObject({ claimed: 1, delivered: 0, superseded: 1 }); + expect(inbox.inserted).toEqual([]); + expect( + outbox.items.get(`member-work-sync:team-a:bob:${status.agenda.fingerprint}`) + ).toMatchObject({ + status: 'superseded', + lastError: 'member_runtime_inactive', + }); + }); + + it('continues dispatching later claimed nudges when one item times out', async () => { + const outbox = new InMemoryOutboxStore(); + const { deps, store } = createDeps({ outboxStore: outbox }); + store.phase2ReadinessState = 'shadow_ready'; + + const status = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const firstItem = [...outbox.items.values()][0]; + expect(firstItem).toBeDefined(); + await outbox.ensurePending({ + id: `${firstItem!.id}:second`, + teamName: firstItem!.teamName, + memberName: firstItem!.memberName, + agendaFingerprint: firstItem!.agendaFingerprint, + payloadHash: `${firstItem!.payloadHash}:second`, + payload: { + ...firstItem!.payload, + workSyncIntentKey: 'test-second', + }, + nowIso: status.evaluatedAt, + }); + + const inserted: Array[0]> = []; + const inbox: MemberWorkSyncInboxNudgePort = { + insertIfAbsent: async (input) => { + if (input.messageId === firstItem!.id) { + return new Promise(() => undefined); + } + inserted.push(input); + return { inserted: true, messageId: input.messageId }; + }, + }; + const dispatcher = new MemberWorkSyncNudgeDispatcher({ + ...deps, + inboxNudge: inbox, + }); + + await expect( + dispatcher.dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + itemTimeoutMs: 1, + }) + ).resolves.toMatchObject({ + claimed: 2, + delivered: 1, + retryable: 1, + }); + + expect(outbox.items.get(firstItem!.id)).toMatchObject({ + status: 'failed_retryable', + lastError: 'nudge dispatch item timed out after 1ms', + }); + expect(inserted).toHaveLength(1); + expect(inserted[0]?.messageId).toBe(`${firstItem!.id}:second`); + expect(outbox.items.get(`${firstItem!.id}:second`)).toMatchObject({ + status: 'delivered', + }); + }); + + it('does not late-deliver an item after item dispatch timeout resolves', async () => { + vi.useFakeTimers(); + try { + const outbox = new InMemoryOutboxStore(); + const { deps, store } = createDeps({ outboxStore: outbox }); + store.phase2ReadinessState = 'shadow_ready'; + + const status = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const firstItem = [...outbox.items.values()][0]; + expect(firstItem).toBeDefined(); + + let resolveInsertStarted!: () => void; + const insertStarted = new Promise((resolve) => { + resolveInsertStarted = resolve; + }); + let resolveInsert!: (value: { inserted: boolean; messageId: string }) => void; + const insertResult = new Promise<{ inserted: boolean; messageId: string }>((resolve) => { + resolveInsert = resolve; + }); + const inbox: MemberWorkSyncInboxNudgePort = { + insertIfAbsent: async () => { + resolveInsertStarted(); + return insertResult; + }, + }; + + const dispatch = new MemberWorkSyncNudgeDispatcher({ + ...deps, + inboxNudge: inbox, + }).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + itemTimeoutMs: 5, + teamTimeoutMs: 100, + }); + await insertStarted; + await vi.advanceTimersByTimeAsync(5); + + await expect(dispatch).resolves.toMatchObject({ + claimed: 1, + delivered: 0, + retryable: 1, + }); + expect(outbox.items.get(firstItem!.id)).toMatchObject({ + status: 'failed_retryable', + lastError: 'nudge dispatch item timed out after 5ms', + }); + + resolveInsert({ inserted: true, messageId: firstItem!.id }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(100); + + expect(outbox.items.get(`member-work-sync:team-a:bob:${status.agenda.fingerprint}`)) + .toMatchObject({ + status: 'failed_retryable', + lastError: 'nudge dispatch item timed out after 5ms', + }); + } finally { + vi.useRealTimers(); + } + }); + + it('continues dispatching later claimed nudges when retry marking also hangs', async () => { + const outbox = new InMemoryOutboxStore(); + const { deps, store } = createDeps({ outboxStore: outbox }); + store.phase2ReadinessState = 'shadow_ready'; + + const status = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const firstItem = [...outbox.items.values()][0]; + expect(firstItem).toBeDefined(); + await outbox.ensurePending({ + id: `${firstItem!.id}:second`, + teamName: firstItem!.teamName, + memberName: firstItem!.memberName, + agendaFingerprint: firstItem!.agendaFingerprint, + payloadHash: `${firstItem!.payloadHash}:second`, + payload: { + ...firstItem!.payload, + workSyncIntentKey: 'test-second', + }, + nowIso: status.evaluatedAt, + }); + + const originalMarkFailed = outbox.markFailed.bind(outbox); + outbox.markFailed = async (input) => { + if (input.id === firstItem!.id) { + return new Promise(() => undefined); + } + return originalMarkFailed(input); + }; + const inserted: Array[0]> = []; + const inbox: MemberWorkSyncInboxNudgePort = { + insertIfAbsent: async (input) => { + if (input.messageId === firstItem!.id) { + return new Promise(() => undefined); + } + inserted.push(input); + return { inserted: true, messageId: input.messageId }; + }, + }; + const dispatcher = new MemberWorkSyncNudgeDispatcher({ + ...deps, + inboxNudge: inbox, + }); + + await expect( + dispatcher.dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + itemTimeoutMs: 1, + }) + ).resolves.toMatchObject({ + claimed: 2, + delivered: 1, + retryable: 1, + }); + + expect(inserted).toHaveLength(1); + expect(inserted[0]?.messageId).toBe(`${firstItem!.id}:second`); + expect(outbox.items.get(`${firstItem!.id}:second`)).toMatchObject({ + status: 'delivered', + }); + }); + + it('continues checking other teams when one team outbox claim hangs', async () => { + vi.useFakeTimers(); + try { + const warn = vi.fn(); + const claimDue = vi.fn( + async (input: Parameters[0]) => { + if (input.teamName === 'stuck') { + await new Promise(() => undefined); + } + return []; + } + ); + const inbox = new InMemoryInboxNudge(); + const { deps } = createDeps({ + outboxStore: { claimDue } as unknown as MemberWorkSyncOutboxStorePort, + inboxNudge: inbox, + }); + deps.logger = { + debug: vi.fn(), + warn, + error: vi.fn(), + }; + + const dispatch = new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['stuck', 'healthy'], + claimedBy: 'test-dispatcher', + claimTimeoutMs: 10, + teamTimeoutMs: 50, + }); + await vi.advanceTimersByTimeAsync(10); + + await expect(dispatch).resolves.toEqual({ + claimed: 0, + delivered: 0, + superseded: 0, + retryable: 0, + terminal: 0, + }); + expect(claimDue).toHaveBeenCalledWith( + expect.objectContaining({ + teamName: 'healthy', + }) + ); + expect(warn).toHaveBeenCalledWith( + 'member work sync nudge claim timed out', + expect.objectContaining({ + teamName: 'stuck', + timeoutMs: 10, + }) + ); + } finally { + vi.useRealTimers(); + } + }); + + it('does not mutate timed-out team items after team dispatch returns', async () => { + vi.useFakeTimers(); + try { + const warn = vi.fn(); + const outbox = new InMemoryOutboxStore(); + const { deps, store } = createDeps({ outboxStore: outbox }); + store.phase2ReadinessState = 'shadow_ready'; + + const status = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const firstItem = [...outbox.items.values()][0]; + expect(firstItem).toBeDefined(); + + let resolveInsertStarted!: () => void; + const insertStarted = new Promise((resolve) => { + resolveInsertStarted = resolve; + }); + let resolveInsert!: (value: { inserted: boolean; messageId: string }) => void; + const insertResult = new Promise<{ inserted: boolean; messageId: string }>((resolve) => { + resolveInsert = resolve; + }); + const inbox: MemberWorkSyncInboxNudgePort = { + insertIfAbsent: async () => { + resolveInsertStarted(); + return insertResult; + }, + }; + deps.logger = { + debug: vi.fn(), + warn, + error: vi.fn(), + }; + + const dispatch = new MemberWorkSyncNudgeDispatcher({ + ...deps, + inboxNudge: inbox, + }).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + itemTimeoutMs: 100, + teamTimeoutMs: 5, + }); + await insertStarted; + await vi.advanceTimersByTimeAsync(5); + + await expect(dispatch).resolves.toEqual({ + claimed: 0, + delivered: 0, + superseded: 0, + retryable: 0, + terminal: 0, + }); + expect(outbox.items.get(firstItem!.id)).toMatchObject({ + status: 'claimed', + }); + expect(warn).toHaveBeenCalledWith( + 'member work sync team nudge dispatch timed out', + expect.objectContaining({ + teamName: 'team-a', + timeoutMs: 5, + }) + ); + + resolveInsert({ inserted: true, messageId: firstItem!.id }); + await Promise.resolve(); + await vi.advanceTimersByTimeAsync(100); + + expect(outbox.items.get(`member-work-sync:team-a:bob:${status.agenda.fingerprint}`)) + .toMatchObject({ + status: 'claimed', + }); + } finally { + vi.useRealTimers(); + } + }); + it('creates a status-only recovery nudge after a delivered nudge turn settles without a report', async () => { const outbox = new InMemoryOutboxStore(); const inbox = new InMemoryInboxNudge(); @@ -902,9 +1394,7 @@ describe('MemberWorkSync use cases', () => { busySignal: { isBusy: async () => { busyChecks += 1; - return busyChecks > 1 - ? { busy: true, reason: 'recent_tool_activity' } - : { busy: false }; + return busyChecks > 1 ? { busy: true, reason: 'recent_tool_activity' } : { busy: false }; }, }, }); @@ -1093,6 +1583,31 @@ describe('MemberWorkSync use cases', () => { expect(recoverySummary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); expect(inbox.inserted).toHaveLength(2); expect(inbox.inserted[1]?.messageId).toContain('agenda-sync-still-stuck'); + + clock.set('2026-04-29T01:02:00.000Z'); + store.metricsGeneratedAt = '2026-04-29T01:02:00.000Z'; + await reconciler.execute( + { + teamName: 'team-a', + memberName: 'team-lead', + }, + { reconciledBy: 'queue', triggerReasons: ['manual_refresh'] } + ); + + const recoveryItems = [...outbox.items.values()].filter((item) => + item.payload.workSyncIntentKey?.startsWith('agenda-sync-still-stuck:') + ); + expect(recoveryItems).toHaveLength(2); + expect(new Set(recoveryItems.map((item) => item.id)).size).toBe(2); + + const secondRecoverySummary = await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(secondRecoverySummary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); + expect(inbox.inserted).toHaveLength(3); + expect(inbox.inserted[2]?.messageId).toContain('agenda-sync-still-stuck'); }); it('creates an agenda-sync refresh recovery when a delivered nudge has a stale payload hash', async () => { @@ -1396,6 +1911,184 @@ describe('MemberWorkSync use cases', () => { expect(inbox.inserted).toHaveLength(3); }); + it('creates a delivered-still-stuck recovery after an accepted still_working lease expires', async () => { + const outbox = new InMemoryOutboxStore(); + const inbox = new InMemoryInboxNudge(); + const { clock, deps, store } = createDeps({ + providerId: 'codex', + outboxStore: outbox, + inboxNudge: inbox, + }); + store.phase2ReadinessState = 'shadow_ready'; + + const reconciler = new MemberWorkSyncReconciler(deps); + const reporter = new MemberWorkSyncReporter(deps); + const firstStatus = await reconciler.execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + const baseId = `member-work-sync:team-a:bob:${firstStatus.agenda.fingerprint}`; + expect(outbox.items.get(baseId)).toMatchObject({ status: 'delivered' }); + + await reporter.execute({ + teamName: 'team-a', + memberName: 'bob', + state: 'still_working', + agendaFingerprint: firstStatus.agenda.fingerprint, + reportToken: firstStatus.reportToken, + taskIds: ['task-1'], + leaseTtlMs: 120_000, + source: 'test', + }); + + clock.set('2026-04-29T00:10:00.000Z'); + store.phase2ReadinessState = 'blocked'; + store.phase2ReadinessReasons = ['would_nudge_rate_high']; + store.metricsGeneratedAt = '2026-04-29T00:10:00.000Z'; + store.recentEvents = [ + { + id: 'old-report-accepted', + teamName: 'team-a', + memberName: 'bob', + kind: 'report_accepted', + state: 'still_working', + agendaFingerprint: firstStatus.agenda.fingerprint, + recordedAt: '2026-04-29T00:01:00.000Z', + actionableCount: 1, + providerId: 'codex', + }, + { + id: 'needs-sync-after-lease-expired', + teamName: 'team-a', + memberName: 'bob', + kind: 'status_evaluated', + state: 'needs_sync', + agendaFingerprint: firstStatus.agenda.fingerprint, + recordedAt: '2026-04-29T00:04:00.000Z', + actionableCount: 1, + providerId: 'codex', + }, + ]; + + const expiredStatus = await reconciler.execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['manual_refresh'] } + ); + + expect(expiredStatus).toMatchObject({ + state: 'needs_sync', + diagnostics: expect.arrayContaining(['report_lease_expired']), + }); + expect(expiredStatus.report).toBeUndefined(); + const recovery = [...outbox.items.values()].find((item) => + item.payload.workSyncIntentKey?.startsWith('agenda-sync-still-stuck:') + ); + expect(recovery).toMatchObject({ + status: 'pending', + agendaFingerprint: firstStatus.agenda.fingerprint, + }); + + const summary = await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(summary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); + expect(inbox.inserted).toHaveLength(2); + expect(inbox.inserted[1]?.messageId).toContain('agenda-sync-still-stuck'); + }); + + it('creates a delivered-still-stuck recovery for mixed review pickup and native work under noisy metrics', async () => { + const outbox = new InMemoryOutboxStore(); + const inbox = new InMemoryInboxNudge(); + const inProgressItem: MemberWorkSyncActionableWorkItem = { + ...workItem, + reason: 'owned_in_progress_task', + evidence: { + status: 'in_progress', + owner: 'bob', + }, + }; + const { clock, deps, store } = createDeps({ + items: [reviewPickupItem, inProgressItem], + providerId: 'codex', + outboxStore: outbox, + inboxNudge: inbox, + }); + store.phase2ReadinessState = 'shadow_ready'; + + const reconciler = new MemberWorkSyncReconciler(deps); + const firstStatus = await reconciler.execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + const baseId = `member-work-sync:team-a:bob:${firstStatus.agenda.fingerprint}`; + expect(outbox.items.get(baseId)).toMatchObject({ status: 'delivered' }); + + clock.set('2026-04-29T00:10:00.000Z'); + store.phase2ReadinessState = 'blocked'; + store.phase2ReadinessReasons = ['would_nudge_rate_high']; + store.metricsGeneratedAt = '2026-04-29T00:10:00.000Z'; + store.recentEvents = [ + { + id: 'mixed-needs-sync-stable', + teamName: 'team-a', + memberName: 'bob', + kind: 'status_evaluated', + state: 'needs_sync', + agendaFingerprint: firstStatus.agenda.fingerprint, + recordedAt: '2026-04-29T00:02:00.000Z', + actionableCount: 2, + providerId: 'codex', + }, + ]; + + await reconciler.execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['manual_refresh'] } + ); + + const recovery = [...outbox.items.values()].find((item) => + item.payload.workSyncIntentKey?.startsWith('agenda-sync-still-stuck:') + ); + expect(recovery).toMatchObject({ + status: 'pending', + agendaFingerprint: firstStatus.agenda.fingerprint, + }); + expect(recovery?.payload.text).toContain('still no accepted member_work_sync_report'); + + const summary = await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(summary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); + expect(inbox.inserted).toHaveLength(2); + expect(inbox.inserted[1]?.messageId).toContain('agenda-sync-still-stuck'); + }); + it('records an existing delivered agenda nudge as skipped before still-stuck recovery age', async () => { const outbox = new InMemoryOutboxStore(); const inbox = new InMemoryInboxNudge(); @@ -1511,6 +2204,33 @@ describe('MemberWorkSync use cases', () => { expect(recoverySummary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); expect(inbox.inserted).toHaveLength(2); expect(inbox.inserted[1]?.messageId).toContain('agenda-sync-still-stuck'); + + clock.set('2026-04-29T01:02:00.000Z'); + store.phase2ReadinessState = 'shadow_ready'; + store.phase2ReadinessReasons = []; + store.metricsGeneratedAt = '2026-04-29T01:02:00.000Z'; + await reconciler.execute( + { + teamName: 'team-a', + memberName: 'team-lead', + }, + { reconciledBy: 'queue', triggerReasons: ['manual_refresh'] } + ); + + const recoveryItems = [...outbox.items.values()].filter((item) => + item.payload.workSyncIntentKey?.startsWith('agenda-sync-still-stuck:') + ); + expect(recoveryItems).toHaveLength(2); + expect(new Set(recoveryItems.map((item) => item.id)).size).toBe(2); + + const secondRecoverySummary = await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(secondRecoverySummary).toMatchObject({ claimed: 1, delivered: 1, retryable: 0 }); + expect(inbox.inserted).toHaveLength(3); + expect(inbox.inserted[2]?.messageId).toContain('agenda-sync-still-stuck'); }); it('creates a still-stuck recovery when a terminal inbox conflict blocks an agenda nudge', async () => { @@ -2096,6 +2816,45 @@ describe('MemberWorkSync use cases', () => { ); }); + it('uses the watchdog cooldown retry deadline instead of exponential retry backoff', async () => { + const outbox = new InMemoryOutboxStore(); + const inbox = new InMemoryInboxNudge(); + const { deps, store } = createDeps({ + outboxStore: outbox, + inboxNudge: inbox, + watchdogCooldown: { + hasRecentNudge: async () => true, + getRecentNudgeCooldown: async () => ({ + active: true, + retryAfterIso: '2026-04-29T00:10:00.000Z', + }), + }, + }); + store.phase2ReadinessState = 'shadow_ready'; + + const current = await new MemberWorkSyncReconciler(deps).execute( + { + teamName: 'team-a', + memberName: 'bob', + }, + { reconciledBy: 'queue', triggerReasons: ['task_changed'] } + ); + const summary = await new MemberWorkSyncNudgeDispatcher(deps).dispatchDue({ + teamNames: ['team-a'], + claimedBy: 'test-dispatcher', + }); + + expect(summary).toMatchObject({ claimed: 1, delivered: 0, retryable: 1 }); + expect(inbox.inserted).toEqual([]); + expect( + outbox.items.get(`member-work-sync:team-a:bob:${current.agenda.fingerprint}`) + ).toMatchObject({ + status: 'failed_retryable', + lastError: 'watchdog_cooldown_active', + nextAttemptAt: '2026-04-29T00:10:00.000Z', + }); + }); + it('uses bounded retry backoff when inbox delivery fails', async () => { const outbox = new InMemoryOutboxStore(); const inbox = new InMemoryInboxNudge(); @@ -2185,4 +2944,41 @@ describe('MemberWorkSync use cases', () => { }); expect(store.writes.at(-1)?.state).toBe('still_working'); }); + + it('supersedes pending controller intents when the member runtime is inactive', async () => { + const { deps, store } = createDeps(); + const reader = new MemberWorkSyncReconciler(deps); + const current = await reader.execute({ teamName: 'team-a', memberName: 'bob' }); + store.pendingIntents.set('intent-1', { + id: 'intent-1', + teamName: 'team-a', + memberName: 'bob', + status: 'pending', + reason: 'control_api_unavailable', + recordedAt: '2026-04-29T00:00:01.000Z', + request: { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working', + agendaFingerprint: current.agenda.fingerprint, + reportToken: current.reportToken, + leaseTtlMs: 120_000, + source: 'mcp', + }, + }); + + const summary = await new MemberWorkSyncPendingReportIntentReplayer({ + ...deps, + lifecycle: { + isTeamActive: () => true, + isMemberActive: () => false, + }, + }).replayTeam('team-a'); + + expect(summary).toEqual({ processed: 1, accepted: 0, rejected: 0, superseded: 1 }); + expect(store.pendingIntents.get('intent-1')).toMatchObject({ + status: 'superseded', + resultCode: 'member_runtime_inactive', + }); + }); }); diff --git a/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts b/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts index 87553f82..b3ab6f7b 100644 --- a/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts +++ b/test/features/member-work-sync/main/JsonMemberWorkSyncStore.test.ts @@ -305,6 +305,107 @@ describe('JsonMemberWorkSyncStore', () => { ).toEqual(['bob', 'tom']); }); + it('repairs a stale processed pending-report index route when member report is pending', async () => { + const request = { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working' as const, + agendaFingerprint: 'agenda:v1:abc', + reportToken: 'wrs:v1.test', + source: 'mcp' as const, + }; + + await store.appendPendingReport(request, 'control_api_unavailable'); + const [intent] = await store.listPendingReports('team-a'); + await store.markPendingReportProcessed('team-a', intent!.id, { + status: 'accepted', + resultCode: 'accepted', + processedAt: '2026-04-29T00:01:00.000Z', + }); + + const reportsPath = join(memberWorkSyncDir(root, 'team-a', 'bob'), 'reports.json'); + const reports = JSON.parse(await readFile(reportsPath, 'utf8')); + reports.intents[intent!.id] = { + ...reports.intents[intent!.id], + status: 'pending', + }; + delete reports.intents[intent!.id].resultCode; + delete reports.intents[intent!.id].processedAt; + await writeFile(reportsPath, JSON.stringify(reports), 'utf8'); + + const pending = await store.listPendingReports('team-a'); + expect(pending).toHaveLength(1); + expect(pending[0]).toMatchObject({ + id: intent!.id, + memberName: 'bob', + status: 'pending', + }); + }); + + it('repairs stale pending-report update routes before marking processed', async () => { + const request = { + teamName: 'team-a', + memberName: 'bob', + state: 'still_working' as const, + agendaFingerprint: 'agenda:v1:abc', + reportToken: 'wrs:v1.test', + source: 'mcp' as const, + }; + + await store.appendPendingReport(request, 'control_api_unavailable'); + await mkdir(memberWorkSyncDir(root, 'team-a', 'tom'), { recursive: true }); + const [intent] = await store.listPendingReports('team-a'); + await writeFile( + join(memberWorkSyncDir(root, 'team-a', 'tom'), 'reports.json'), + JSON.stringify({ + schemaVersion: 2, + intents: { + [intent!.id]: { + ...intent!, + teamName: 'other-team', + memberName: 'tom', + }, + }, + }), + 'utf8' + ); + const indexPath = join( + root, + 'team-a', + '.member-work-sync', + 'indexes', + 'pending-reports-index.json' + ); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + index.items[intent!.id] = { + ...index.items[intent!.id], + memberKey: 'tom', + memberName: 'tom', + }; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + await store.markPendingReportProcessed('team-a', intent!.id, { + status: 'accepted', + resultCode: 'accepted', + processedAt: '2026-04-29T00:01:00.000Z', + }); + + const reports = JSON.parse( + await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'reports.json'), 'utf8') + ); + expect(reports.intents[intent!.id]).toMatchObject({ + memberName: 'bob', + status: 'accepted', + resultCode: 'accepted', + }); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect(repaired.items[intent!.id]).toMatchObject({ + memberKey: 'bob', + memberName: 'bob', + status: 'accepted', + }); + }); + it('records bounded shadow metrics from status writes', async () => { await store.write(makeStatus({})); await store.write( @@ -611,6 +712,60 @@ describe('JsonMemberWorkSyncStore', () => { ).resolves.toEqual([]); }); + it('treats invalid retry delay timestamps as due so retryable items cannot sleep forever', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markFailed({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + retryable: true, + error: 'member_busy:active_tool_activity', + nextAttemptAt: '2026-04-29T00:30:00.000Z', + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const memberOutboxPath = join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'); + const memberOutbox = JSON.parse(await readFile(memberOutboxPath, 'utf8')); + memberOutbox.items[input.id].nextAttemptAt = 'not-a-date'; + await writeFile(memberOutboxPath, JSON.stringify(memberOutbox), 'utf8'); + + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + index.items[input.id].nextAttemptAt = 'not-a-date'; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + await expect( + store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-b', + nowIso: '2026-04-29T00:04:00.000Z', + limit: 1, + }) + ).resolves.toEqual([ + expect.objectContaining({ + id: input.id, + status: 'claimed', + attemptGeneration: claimed.attemptGeneration + 1, + }), + ]); + }); + it('clears retry delay when a retryable outbox item is delivered', async () => { const input = { id: 'member-work-sync:team-a:bob:agenda:v1:abc', @@ -672,6 +827,185 @@ describe('JsonMemberWorkSyncStore', () => { expect(index.items[input.id]).not.toHaveProperty('nextAttemptAt'); }); + it('keeps delivered outbox items delivered when a late retry mark races after delivery', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markDelivered({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + deliveredMessageId: 'message-1', + nowIso: '2026-04-29T00:01:30.000Z', + }); + await store.markFailed({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + retryable: true, + error: 'nudge dispatch item timed out after 1ms', + nextAttemptAt: '2026-04-29T00:03:00.000Z', + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const memberOutbox = JSON.parse( + await readFile( + join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'outbox.json'), + 'utf8' + ) + ); + expect(memberOutbox.items[input.id]).toMatchObject({ + status: 'delivered', + deliveredMessageId: 'message-1', + }); + expect(memberOutbox.items[input.id]).not.toHaveProperty('lastError'); + expect(memberOutbox.items[input.id]).not.toHaveProperty('nextAttemptAt'); + + const index = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), + 'utf8' + ) + ); + expect(index.items[input.id]).toMatchObject({ + status: 'delivered', + }); + expect(index.items[input.id]).not.toHaveProperty('nextAttemptAt'); + }); + + it('keeps retryable outbox items retryable when a late delivery races after timeout', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:retry-race', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:retry-race', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markFailed({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + retryable: true, + error: 'nudge dispatch item timed out after 1ms', + nextAttemptAt: '2026-04-29T00:03:00.000Z', + nowIso: '2026-04-29T00:02:00.000Z', + }); + await store.markDelivered({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + deliveredMessageId: 'late-message', + nowIso: '2026-04-29T00:02:30.000Z', + }); + + const memberOutbox = JSON.parse( + await readFile( + join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'outbox.json'), + 'utf8' + ) + ); + expect(memberOutbox.items[input.id]).toMatchObject({ + status: 'failed_retryable', + lastError: 'nudge dispatch item timed out after 1ms', + nextAttemptAt: '2026-04-29T00:03:00.000Z', + }); + expect(memberOutbox.items[input.id]).not.toHaveProperty('deliveredMessageId'); + + const index = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), + 'utf8' + ) + ); + expect(index.items[input.id]).toMatchObject({ + status: 'failed_retryable', + nextAttemptAt: '2026-04-29T00:03:00.000Z', + }); + expect(index.items[input.id]).not.toHaveProperty('deliveredMessageId'); + }); + + it('keeps terminal outbox items terminal when a late delivery races after failure', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:terminal-race', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:terminal-race', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markFailed({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + retryable: false, + error: 'inbox_payload_conflict', + nowIso: '2026-04-29T00:01:30.000Z', + }); + await store.markDelivered({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed.attemptGeneration, + deliveredMessageId: 'late-message', + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const memberOutbox = JSON.parse( + await readFile( + join(root, 'team-a', 'members', 'bob', '.member-work-sync', 'outbox.json'), + 'utf8' + ) + ); + expect(memberOutbox.items[input.id]).toMatchObject({ + status: 'failed_terminal', + lastError: 'inbox_payload_conflict', + }); + expect(memberOutbox.items[input.id]).not.toHaveProperty('deliveredMessageId'); + + const index = JSON.parse( + await readFile( + join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'), + 'utf8' + ) + ); + expect(index.items[input.id]).toMatchObject({ + status: 'failed_terminal', + }); + expect(index.items[input.id]).not.toHaveProperty('deliveredMessageId'); + }); + it('finds recent recovery outbox rows by logical intent key', async () => { const olderInput = { id: 'member-work-sync:team-a:bob:agenda:v1:older', @@ -761,6 +1095,34 @@ describe('JsonMemberWorkSyncStore', () => { ).resolves.toBeNull(); }); + it('ignores superseded rows for logical recovery lookup', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:superseded', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:superseded', + payloadHash: 'hash-a', + payload: makeNudgePayload({ workSyncIntentKey: 'proof-missing:message-1' }), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + await store.markSuperseded({ + teamName: 'team-a', + id: input.id, + reason: 'status_no_longer_matches_outbox', + nowIso: '2026-04-29T00:01:00.000Z', + }); + + await expect( + store.findRecentRecoveryByIntent({ + teamName: 'team-a', + memberName: 'bob', + intentKey: 'proof-missing:message-1', + sinceIso: '2026-04-29T00:00:00.000Z', + }) + ).resolves.toBeNull(); + }); + it('claims due outbox items and fences terminal updates by attempt generation', async () => { const input = { id: 'member-work-sync:team-a:bob:agenda:v1:abc', @@ -838,6 +1200,98 @@ describe('JsonMemberWorkSyncStore', () => { }); }); + it('reclaims stale claimed outbox items without waiting for a fresh reconcile', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:stale-claim', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:stale-claim', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + expect(claimed).toMatchObject({ + id: input.id, + status: 'claimed', + attemptGeneration: 1, + claimedBy: 'dispatcher-a', + claimedAt: '2026-04-29T00:01:00.000Z', + }); + + await expect( + store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-b', + nowIso: '2026-04-29T00:05:59.000Z', + limit: 1, + }) + ).resolves.toEqual([]); + + const [reclaimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-b', + nowIso: '2026-04-29T00:06:00.000Z', + limit: 1, + }); + expect(reclaimed).toMatchObject({ + id: input.id, + status: 'claimed', + attemptGeneration: 2, + claimedBy: 'dispatcher-b', + claimedAt: '2026-04-29T00:06:00.000Z', + }); + }); + + it('treats future claimedAt outbox items as stale', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:future-claim', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:future-claim', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:10:00.000Z', + limit: 1, + }); + expect(claimed).toMatchObject({ + id: input.id, + status: 'claimed', + attemptGeneration: 1, + claimedBy: 'dispatcher-a', + claimedAt: '2026-04-29T00:10:00.000Z', + }); + + const [reclaimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-b', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + + expect(reclaimed).toMatchObject({ + id: input.id, + status: 'claimed', + attemptGeneration: 2, + claimedBy: 'dispatcher-b', + claimedAt: '2026-04-29T00:01:00.000Z', + }); + }); + it('claims due outbox items from the index without scanning unrelated member outboxes', async () => { const bobInput = { id: 'member-work-sync:team-a:bob:agenda:v1:abc', @@ -1210,6 +1664,221 @@ describe('JsonMemberWorkSyncStore', () => { expect(claimed.map((item) => item.memberName).sort()).toEqual(['bob', 'tom']); }); + it('rewrites stale due outbox member keys while claiming', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + index.items[input.id] = { + ...index.items[input.id], + memberKey: 'tom', + memberName: 'bob', + }; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + + expect(claimed).toMatchObject({ + id: input.id, + memberName: 'bob', + status: 'claimed', + }); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect(repaired.items[input.id]).toMatchObject({ + memberKey: 'bob', + memberName: 'bob', + status: 'claimed', + }); + }); + + it('repairs stale outbox update routes before marking failures', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + await mkdir(memberWorkSyncDir(root, 'team-a', 'tom'), { recursive: true }); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await writeFile( + join(memberWorkSyncDir(root, 'team-a', 'tom'), 'outbox.json'), + JSON.stringify({ + schemaVersion: 2, + items: { + [input.id]: { + ...input, + teamName: 'other-team', + memberName: 'tom', + status: 'claimed', + attemptGeneration: claimed!.attemptGeneration, + claimedBy: 'dispatcher-a', + claimedAt: '2026-04-29T00:01:00.000Z', + updatedAt: '2026-04-29T00:01:00.000Z', + }, + }, + }), + 'utf8' + ); + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + index.items[input.id] = { + ...index.items[input.id], + memberKey: 'tom', + memberName: 'tom', + }; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + await store.markFailed({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed!.attemptGeneration, + error: 'delivery failed', + retryable: true, + nextAttemptAt: '2026-04-29T00:10:00.000Z', + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const memberOutbox = JSON.parse( + await readFile(join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'), 'utf8') + ); + expect(memberOutbox.items[input.id]).toMatchObject({ + status: 'failed_retryable', + lastError: 'delivery failed', + nextAttemptAt: '2026-04-29T00:10:00.000Z', + }); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect(repaired.items[input.id]).toMatchObject({ + memberKey: 'bob', + memberName: 'bob', + status: 'failed_retryable', + }); + }); + + it('repairs wrong-member due outbox index routes before returning a limited claim', async () => { + const bobInput = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(bobInput); + await mkdir(memberWorkSyncDir(root, 'team-a', 'tom'), { recursive: true }); + await writeFile( + join(memberWorkSyncDir(root, 'team-a', 'tom'), 'outbox.json'), + JSON.stringify({ + schemaVersion: 2, + items: { + [bobInput.id]: { + ...bobInput, + teamName: 'other-team', + memberName: 'tom', + status: 'pending', + createdAt: '2026-04-29T00:00:00.000Z', + updatedAt: '2026-04-29T00:00:00.000Z', + }, + }, + }), + 'utf8' + ); + const indexPath = join(root, 'team-a', '.member-work-sync', 'indexes', 'outbox-index.json'); + const index = JSON.parse(await readFile(indexPath, 'utf8')); + index.items[bobInput.id] = { + ...index.items[bobInput.id], + memberKey: 'tom', + memberName: 'tom', + }; + await writeFile(indexPath, JSON.stringify(index), 'utf8'); + + const claimed = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + + expect(claimed.map((item) => item.memberName)).toEqual(['bob']); + const repaired = JSON.parse(await readFile(indexPath, 'utf8')); + expect(repaired.items[bobInput.id]).toMatchObject({ + memberKey: 'bob', + memberName: 'bob', + status: 'claimed', + }); + }); + + it('repairs stale terminal outbox index routes when member-scoped item is due', async () => { + const input = { + id: 'member-work-sync:team-a:bob:agenda:v1:abc', + teamName: 'team-a', + memberName: 'bob', + agendaFingerprint: 'agenda:v1:abc', + payloadHash: 'hash-a', + payload: makeNudgePayload(), + nowIso: '2026-04-29T00:00:00.000Z', + }; + await store.ensurePending(input); + const [claimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-a', + nowIso: '2026-04-29T00:01:00.000Z', + limit: 1, + }); + await store.markDelivered({ + teamName: 'team-a', + id: input.id, + attemptGeneration: claimed!.attemptGeneration, + deliveredMessageId: input.id, + nowIso: '2026-04-29T00:02:00.000Z', + }); + + const memberOutboxPath = join(memberWorkSyncDir(root, 'team-a', 'bob'), 'outbox.json'); + const memberOutbox = JSON.parse(await readFile(memberOutboxPath, 'utf8')); + memberOutbox.items[input.id] = { + ...memberOutbox.items[input.id], + status: 'pending', + updatedAt: '2026-04-29T00:03:00.000Z', + }; + delete memberOutbox.items[input.id].deliveredMessageId; + await writeFile(memberOutboxPath, JSON.stringify(memberOutbox), 'utf8'); + + const [reclaimed] = await store.claimDue({ + teamName: 'team-a', + claimedBy: 'dispatcher-b', + nowIso: '2026-04-29T00:04:00.000Z', + limit: 1, + }); + expect(reclaimed).toMatchObject({ + id: input.id, + status: 'claimed', + attemptGeneration: 2, + claimedBy: 'dispatcher-b', + }); + }); + it('falls back to legacy v1 status and materializes legacy outbox during claim', async () => { const auditEvents: MemberWorkSyncAuditEvent[] = []; store = new JsonMemberWorkSyncStore(new MemberWorkSyncStorePaths(root), { diff --git a/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts b/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts index 9185af3a..50407e41 100644 --- a/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts +++ b/test/features/member-work-sync/main/MemberWorkSyncEventQueue.test.ts @@ -1,6 +1,5 @@ -import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; - import { MemberWorkSyncEventQueue } from '@features/member-work-sync/main/infrastructure/MemberWorkSyncEventQueue'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; describe('MemberWorkSyncEventQueue', () => { beforeEach(() => { @@ -230,6 +229,29 @@ describe('MemberWorkSyncEventQueue', () => { await queue.stop(); }); + it('can reconcile inactive teams when the caller needs inactive statuses refreshed', async () => { + const reconcile = vi.fn(async () => undefined); + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + reconcile, + isTeamActive: () => false, + reconcileInactiveTeams: true, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'manual_refresh' }); + await vi.advanceTimersByTimeAsync(1); + + expect(reconcile).toHaveBeenCalledWith( + { teamName: 'team-a', memberName: 'bob' }, + expect.objectContaining({ + reconciledBy: 'queue', + triggerReasons: ['manual_refresh'], + }) + ); + await queue.stop(); + expect(queue.getDiagnostics()).toMatchObject({ dropped: 0, reconciled: 1 }); + }); + it('runs one follow-up pass when events arrive during an active reconcile', async () => { let release: () => void = () => { throw new Error('reconcile did not start'); @@ -370,4 +392,210 @@ describe('MemberWorkSyncEventQueue', () => { expect(reconciles).toHaveLength(2); await queue.stop(); }); + + it('retries a failed reconcile with bounded backoff', async () => { + const reconciles: unknown[] = []; + const auditEvents: string[] = []; + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + retryDelayMs: 10, + maxRetryAttempts: 2, + reconcile: async (request) => { + reconciles.push(request); + if (reconciles.length === 1) { + throw new Error('transient'); + } + }, + isTeamActive: () => true, + auditJournal: { + append: async (event) => { + auditEvents.push(event.event); + }, + }, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'turn_settled' }); + await vi.advanceTimersByTimeAsync(1); + + expect(reconciles).toHaveLength(1); + expect(queue.getDiagnostics()).toMatchObject({ failed: 1, queued: 1, reconciled: 0 }); + expect(auditEvents).toEqual(['queue_enqueued', 'queue_retry_scheduled']); + + await vi.advanceTimersByTimeAsync(9); + expect(reconciles).toHaveLength(1); + + await vi.advanceTimersByTimeAsync(1); + expect(reconciles).toHaveLength(2); + expect(queue.getDiagnostics()).toMatchObject({ failed: 1, queued: 0, reconciled: 1 }); + + await queue.stop(); + }); + + it('times out a hung reconcile and retries so the member cannot stay running forever', async () => { + let reconcileCalls = 0; + const auditEvents: string[] = []; + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + retryDelayMs: 10, + reconcileTimeoutMs: 20, + maxRetryAttempts: 1, + reconcile: async () => { + reconcileCalls += 1; + if (reconcileCalls === 1) { + await new Promise(() => undefined); + } + }, + isTeamActive: () => true, + auditJournal: { + append: async (event) => { + auditEvents.push(event.event); + }, + }, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'turn_settled' }); + await vi.advanceTimersByTimeAsync(1); + + expect(reconcileCalls).toBe(1); + expect(queue.getDiagnostics()).toMatchObject({ running: 1, queued: 0, failed: 0 }); + + await vi.advanceTimersByTimeAsync(20); + + expect(queue.getDiagnostics()).toMatchObject({ + running: 0, + queued: 1, + failed: 1, + reconciled: 0, + }); + expect(auditEvents).toEqual(['queue_enqueued', 'queue_retry_scheduled']); + + await vi.advanceTimersByTimeAsync(9); + expect(reconcileCalls).toBe(1); + + await vi.advanceTimersByTimeAsync(1); + + expect(reconcileCalls).toBe(2); + expect(queue.getDiagnostics()).toMatchObject({ + running: 0, + queued: 0, + failed: 1, + reconciled: 1, + }); + expect(auditEvents).toEqual([ + 'queue_enqueued', + 'queue_retry_scheduled', + 'queue_reconciled', + ]); + + await queue.stop(); + }); + + it('marks a timed-out reconcile context as cancelled for late continuations', async () => { + let releaseFirst: () => void = () => { + throw new Error('first reconcile did not start'); + }; + let reconcileCalls = 0; + const lateSideEffects: string[] = []; + const cancellationChecks: boolean[] = []; + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + retryDelayMs: 10, + reconcileTimeoutMs: 20, + maxRetryAttempts: 1, + reconcile: async (_request, context) => { + reconcileCalls += 1; + if (reconcileCalls === 1) { + await new Promise((resolve) => { + releaseFirst = resolve; + }); + const cancelled = context.isCancelled?.() === true; + cancellationChecks.push(cancelled); + if (!cancelled) { + lateSideEffects.push('first'); + } + return; + } + + const cancelled = context.isCancelled?.() === true; + cancellationChecks.push(cancelled); + if (!cancelled) { + lateSideEffects.push('retry'); + } + }, + isTeamActive: () => true, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'turn_settled' }); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(20); + await vi.advanceTimersByTimeAsync(10); + + expect(lateSideEffects).toEqual(['retry']); + + releaseFirst(); + await vi.advanceTimersByTimeAsync(1); + + expect(cancellationChecks).toEqual([false, true]); + expect(lateSideEffects).toEqual(['retry']); + + await queue.stop(); + }); + + it('drops a failed reconcile after the retry budget is exhausted', async () => { + const reconcile = vi.fn(async () => { + throw new Error('still failing'); + }); + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + retryDelayMs: 10, + maxRetryAttempts: 1, + reconcile, + isTeamActive: () => true, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'turn_settled' }); + await vi.advanceTimersByTimeAsync(1); + await vi.advanceTimersByTimeAsync(10); + await vi.advanceTimersByTimeAsync(1_000); + + expect(reconcile).toHaveBeenCalledTimes(2); + expect(queue.getDiagnostics()).toMatchObject({ + dropped: 1, + failed: 2, + queued: 0, + reconciled: 0, + }); + + await queue.stop(); + }); + + it('resets retry budget when a fresh event joins a queued retry item', async () => { + const reconcile = vi.fn(async () => { + throw new Error('still failing'); + }); + const queue = new MemberWorkSyncEventQueue({ + quietWindowMs: 1, + retryDelayMs: 10, + maxRetryAttempts: 1, + reconcile, + isTeamActive: () => true, + }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'turn_settled' }); + await vi.advanceTimersByTimeAsync(1); + expect(queue.getDiagnostics()).toMatchObject({ failed: 1, queued: 1, dropped: 0 }); + + queue.enqueue({ teamName: 'team-a', memberName: 'bob', triggerReason: 'task_changed' }); + await vi.advanceTimersByTimeAsync(10); + + expect(reconcile).toHaveBeenCalledTimes(2); + expect(queue.getDiagnostics()).toMatchObject({ + dropped: 0, + failed: 2, + queued: 1, + reconciled: 0, + }); + + await queue.stop(); + }); }); diff --git a/test/features/member-work-sync/main/MemberWorkSyncNudgeDispatchScheduler.test.ts b/test/features/member-work-sync/main/MemberWorkSyncNudgeDispatchScheduler.test.ts index 90cc0a30..692dcb7b 100644 --- a/test/features/member-work-sync/main/MemberWorkSyncNudgeDispatchScheduler.test.ts +++ b/test/features/member-work-sync/main/MemberWorkSyncNudgeDispatchScheduler.test.ts @@ -1,6 +1,5 @@ -import { describe, expect, it, vi } from 'vitest'; - import { MemberWorkSyncNudgeDispatchScheduler } from '@features/member-work-sync/main/infrastructure/MemberWorkSyncNudgeDispatchScheduler'; +import { describe, expect, it, vi } from 'vitest'; describe('MemberWorkSyncNudgeDispatchScheduler', () => { it('dispatches due nudges for unique active teams without overlapping runs', async () => { @@ -19,7 +18,9 @@ describe('MemberWorkSyncNudgeDispatchScheduler', () => { const first = scheduler.runOnce(); const second = scheduler.runOnce(); - await Promise.resolve(); + await new Promise((resolve) => { + setTimeout(resolve, 0); + }); expect(dispatchDue).toHaveBeenCalledTimes(1); release(); @@ -61,4 +62,99 @@ describe('MemberWorkSyncNudgeDispatchScheduler', () => { expect.objectContaining({ error: 'Error: list failed' }) ); }); + + it('times out a hung dispatch so later scheduled runs can continue', async () => { + vi.useFakeTimers(); + try { + let dispatchCalls = 0; + const warn = vi.fn(); + const dispatchDue = vi.fn(async () => { + dispatchCalls += 1; + if (dispatchCalls === 1) { + await new Promise(() => undefined); + } + return { claimed: 0, delivered: 0, superseded: 0, retryable: 0, terminal: 0 }; + }); + const scheduler = new MemberWorkSyncNudgeDispatchScheduler({ + listLifecycleActiveTeamNames: async () => ['team-a'], + dispatchDue, + dispatchTimeoutMs: 20, + logger: { + debug: vi.fn(), + warn, + error: vi.fn(), + }, + }); + + const first = scheduler.runOnce(); + await vi.advanceTimersByTimeAsync(0); + + expect(dispatchDue).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(20); + await first; + + expect(warn).toHaveBeenCalledWith( + 'member work sync scheduled nudge dispatch failed', + expect.objectContaining({ + error: 'Error: member work sync scheduled nudge dispatch timed out after 20ms', + }) + ); + + await scheduler.runOnce(); + + expect(dispatchDue).toHaveBeenCalledTimes(2); + } finally { + vi.useRealTimers(); + } + }); + + it('times out hung active team listing so later scheduled runs can continue', async () => { + vi.useFakeTimers(); + try { + let listCalls = 0; + const warn = vi.fn(); + const dispatchDue = vi.fn(async () => ({ + claimed: 0, + delivered: 0, + superseded: 0, + retryable: 0, + terminal: 0, + })); + const scheduler = new MemberWorkSyncNudgeDispatchScheduler({ + listLifecycleActiveTeamNames: async () => { + listCalls += 1; + if (listCalls === 1) { + await new Promise(() => undefined); + } + return ['team-a']; + }, + dispatchDue, + dispatchTimeoutMs: 20, + logger: { + debug: vi.fn(), + warn, + error: vi.fn(), + }, + }); + + const first = scheduler.runOnce(); + await vi.advanceTimersByTimeAsync(20); + await first; + + expect(warn).toHaveBeenCalledWith( + 'member work sync scheduled nudge dispatch failed', + expect.objectContaining({ + error: 'Error: member work sync scheduled nudge team listing timed out after 20ms', + }) + ); + expect(dispatchDue).not.toHaveBeenCalled(); + + await scheduler.runOnce(); + + expect(dispatchDue).toHaveBeenCalledWith(['team-a']); + } finally { + vi.useRealTimers(); + } + }); }); diff --git a/test/features/member-work-sync/main/MemberWorkSyncTeamChangeRouter.test.ts b/test/features/member-work-sync/main/MemberWorkSyncTeamChangeRouter.test.ts index 59f0038d..d4625eec 100644 --- a/test/features/member-work-sync/main/MemberWorkSyncTeamChangeRouter.test.ts +++ b/test/features/member-work-sync/main/MemberWorkSyncTeamChangeRouter.test.ts @@ -1,6 +1,5 @@ -import { describe, expect, it, vi } from 'vitest'; - import { MemberWorkSyncTeamChangeRouter } from '@features/member-work-sync/main/adapters/input/MemberWorkSyncTeamChangeRouter'; +import { describe, expect, it, vi } from 'vitest'; function createRouter(activeMembers: string[] = ['alice', 'bob']) { const queue = { @@ -96,13 +95,25 @@ describe('MemberWorkSyncTeamChangeRouter', () => { }); }); - it('drops queued work when the team goes offline', () => { + it('refreshes member runtime state when the team goes offline', async () => { const { queue, router } = createRouter(); router.noteTeamChange({ type: 'lead-activity', teamName: 'team-a', detail: 'offline' }); + await Promise.resolve(); expect(queue.dropTeam).toHaveBeenCalledWith('team-a'); - expect(queue.enqueue).not.toHaveBeenCalled(); + expect(queue.enqueue).toHaveBeenCalledWith({ + teamName: 'team-a', + memberName: 'alice', + triggerReason: 'runtime_activity', + runAfterMs: 0, + }); + expect(queue.enqueue).toHaveBeenCalledWith({ + teamName: 'team-a', + memberName: 'bob', + triggerReason: 'runtime_activity', + runAfterMs: 0, + }); }); it('routes member-turn-settled events to one member reconcile', () => { diff --git a/test/features/member-work-sync/main/MemberWorkSyncToolActivityBusySignal.test.ts b/test/features/member-work-sync/main/MemberWorkSyncToolActivityBusySignal.test.ts index 9c736020..8be733e9 100644 --- a/test/features/member-work-sync/main/MemberWorkSyncToolActivityBusySignal.test.ts +++ b/test/features/member-work-sync/main/MemberWorkSyncToolActivityBusySignal.test.ts @@ -1,5 +1,5 @@ import { MemberWorkSyncToolActivityBusySignal } from '@features/member-work-sync/main/infrastructure/MemberWorkSyncToolActivityBusySignal'; -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import type { TeamChangeEvent, ToolActivityEventPayload } from '@shared/types'; @@ -180,4 +180,80 @@ describe('MemberWorkSyncToolActivityBusySignal', () => { }) ).resolves.toEqual({ busy: false }); }); + + it('bounds future tool timestamps so busy state cannot sleep nudges for too long', async () => { + vi.useFakeTimers(); + try { + vi.setSystemTime(new Date('2026-04-29T00:00:00.000Z')); + + const activeSignal = new MemberWorkSyncToolActivityBusySignal({ + busyGraceMs: 90_000, + activeToolStaleMs: 10 * 60_000, + }); + + activeSignal.noteTeamChange( + toolEvent('team-a', { + action: 'start', + activity: { + memberName: 'bob', + toolUseId: 'tool-1', + toolName: 'bash', + startedAt: '2026-04-29T01:00:00.000Z', + source: 'runtime', + }, + }) + ); + + await expect( + activeSignal.isBusy({ + teamName: 'team-a', + memberName: 'bob', + nowIso: '2026-04-29T00:09:59.000Z', + }) + ).resolves.toMatchObject({ + busy: true, + reason: 'active_tool_activity', + }); + + await expect( + activeSignal.isBusy({ + teamName: 'team-a', + memberName: 'bob', + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toEqual({ busy: false }); + + const finishSignal = new MemberWorkSyncToolActivityBusySignal({ busyGraceMs: 90_000 }); + finishSignal.noteTeamChange( + toolEvent('team-a', { + action: 'finish', + memberName: 'bob', + toolUseId: 'tool-2', + finishedAt: '2026-04-29T01:00:00.000Z', + }) + ); + + await expect( + finishSignal.isBusy({ + teamName: 'team-a', + memberName: 'bob', + nowIso: '2026-04-29T00:01:29.000Z', + }) + ).resolves.toMatchObject({ + busy: true, + reason: 'recent_tool_activity', + retryAfterIso: '2026-04-29T00:01:30.000Z', + }); + + await expect( + finishSignal.isBusy({ + teamName: 'team-a', + memberName: 'bob', + nowIso: '2026-04-29T00:01:30.000Z', + }) + ).resolves.toEqual({ busy: false }); + } finally { + vi.useRealTimers(); + } + }); }); diff --git a/test/features/member-work-sync/main/RuntimeTurnSettledDrainScheduler.test.ts b/test/features/member-work-sync/main/RuntimeTurnSettledDrainScheduler.test.ts new file mode 100644 index 00000000..9c0fcc96 --- /dev/null +++ b/test/features/member-work-sync/main/RuntimeTurnSettledDrainScheduler.test.ts @@ -0,0 +1,75 @@ +import { RuntimeTurnSettledDrainScheduler } from '@features/member-work-sync/main/infrastructure/RuntimeTurnSettledDrainScheduler'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +describe('RuntimeTurnSettledDrainScheduler', () => { + beforeEach(() => { + vi.useFakeTimers(); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + it('does not overlap active drains', async () => { + let release!: () => void; + const firstDrain = new Promise((resolve) => { + release = resolve; + }); + const drain = vi.fn(async () => { + await firstDrain; + return { claimed: 1, enqueued: 1, unresolved: 0, ignored: 0, invalid: 0, failed: 0 }; + }); + const scheduler = new RuntimeTurnSettledDrainScheduler({ drain }); + + const first = scheduler.drainNow(); + await vi.advanceTimersByTimeAsync(0); + + await expect(scheduler.drainNow()).resolves.toBeNull(); + expect(drain).toHaveBeenCalledTimes(1); + + release(); + await first; + }); + + it('times out a hung drain so later turn-settled drains can continue', async () => { + let drainCalls = 0; + const warn = vi.fn(); + const drain = vi.fn(async () => { + drainCalls += 1; + if (drainCalls === 1) { + await new Promise(() => undefined); + } + return { claimed: 0, enqueued: 0, unresolved: 0, ignored: 0, invalid: 0, failed: 0 }; + }); + const scheduler = new RuntimeTurnSettledDrainScheduler({ + drain, + drainTimeoutMs: 20, + logger: { + debug: vi.fn(), + warn, + error: vi.fn(), + }, + }); + + const first = scheduler.drainNow(); + await vi.advanceTimersByTimeAsync(0); + + expect(drain).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(20); + await expect(first).resolves.toBeNull(); + + expect(warn).toHaveBeenCalledWith( + 'runtime turn settled scheduled drain failed', + expect.objectContaining({ + error: 'Error: runtime turn settled drain timed out after 20ms', + }) + ); + + await expect(scheduler.drainNow()).resolves.toMatchObject({ + claimed: 0, + enqueued: 0, + }); + expect(drain).toHaveBeenCalledTimes(2); + }); +}); diff --git a/test/features/member-work-sync/main/TeamTaskStallJournalWorkSyncCooldown.test.ts b/test/features/member-work-sync/main/TeamTaskStallJournalWorkSyncCooldown.test.ts index 6dba26db..1e264f30 100644 --- a/test/features/member-work-sync/main/TeamTaskStallJournalWorkSyncCooldown.test.ts +++ b/test/features/member-work-sync/main/TeamTaskStallJournalWorkSyncCooldown.test.ts @@ -41,6 +41,71 @@ describe('TeamTaskStallJournalWorkSyncCooldown', () => { ).resolves.toBe(true); }); + it('returns the exact retry deadline for an active watchdog cooldown', async () => { + await mkdir(join(root, 'team-a'), { recursive: true }); + await writeFile( + join(root, 'team-a', 'stall-monitor-journal.json'), + JSON.stringify([ + { + taskId: 'task-1', + memberName: 'bob', + state: 'alerted', + alertedAt: '2026-04-29T00:05:00.000Z', + }, + ]), + 'utf8' + ); + + const cooldown = new TeamTaskStallJournalWorkSyncCooldown(root, 10 * 60_000); + + await expect( + cooldown.getRecentNudgeCooldown({ + teamName: 'team-a', + memberName: 'bob', + taskIds: ['task-1'], + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toEqual({ + active: true, + retryAfterIso: '2026-04-29T00:15:00.000Z', + }); + }); + + it('does not suppress a reassigned task for a different member', async () => { + await mkdir(join(root, 'team-a'), { recursive: true }); + await writeFile( + join(root, 'team-a', 'stall-monitor-journal.json'), + JSON.stringify([ + { + taskId: 'task-1', + memberName: 'alice', + state: 'alerted', + alertedAt: '2026-04-29T00:05:00.000Z', + }, + ]), + 'utf8' + ); + + const cooldown = new TeamTaskStallJournalWorkSyncCooldown(root, 10 * 60_000); + + await expect( + cooldown.hasRecentNudge({ + teamName: 'team-a', + memberName: 'bob', + taskIds: ['task-1'], + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toBe(false); + await expect( + cooldown.hasRecentNudge({ + teamName: 'team-a', + memberName: 'alice', + taskIds: ['task-1'], + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toBe(true); + }); + it('ignores old watchdog alerts and missing journals', async () => { await mkdir(join(root, 'team-a'), { recursive: true }); await writeFile( @@ -75,6 +140,58 @@ describe('TeamTaskStallJournalWorkSyncCooldown', () => { ).resolves.toBe(false); }); + it('does not suppress exactly at the watchdog cooldown boundary', async () => { + await mkdir(join(root, 'team-a'), { recursive: true }); + await writeFile( + join(root, 'team-a', 'stall-monitor-journal.json'), + JSON.stringify([ + { + taskId: 'task-1', + state: 'alerted', + alertedAt: '2026-04-29T00:00:00.000Z', + }, + ]), + 'utf8' + ); + + const cooldown = new TeamTaskStallJournalWorkSyncCooldown(root, 10 * 60_000); + + await expect( + cooldown.getRecentNudgeCooldown({ + teamName: 'team-a', + memberName: 'bob', + taskIds: ['task-1'], + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toEqual({ active: false }); + }); + + it('ignores future watchdog alert timestamps', async () => { + await mkdir(join(root, 'team-a'), { recursive: true }); + await writeFile( + join(root, 'team-a', 'stall-monitor-journal.json'), + JSON.stringify([ + { + taskId: 'task-1', + state: 'alerted', + alertedAt: '2026-04-29T01:00:00.000Z', + }, + ]), + 'utf8' + ); + + const cooldown = new TeamTaskStallJournalWorkSyncCooldown(root, 10 * 60_000); + + await expect( + cooldown.hasRecentNudge({ + teamName: 'team-a', + memberName: 'bob', + taskIds: ['task-1'], + nowIso: '2026-04-29T00:10:00.000Z', + }) + ).resolves.toBe(false); + }); + it('fails open when the watchdog journal is invalid', async () => { await mkdir(join(root, 'team-a'), { recursive: true }); await writeFile(join(root, 'team-a', 'stall-monitor-journal.json'), '{bad json', 'utf8'); diff --git a/test/features/member-work-sync/main/createMemberWorkSyncFeature.test.ts b/test/features/member-work-sync/main/createMemberWorkSyncFeature.test.ts index 3f3b80aa..45a4d82c 100644 --- a/test/features/member-work-sync/main/createMemberWorkSyncFeature.test.ts +++ b/test/features/member-work-sync/main/createMemberWorkSyncFeature.test.ts @@ -651,6 +651,95 @@ describe('createMemberWorkSyncFeature composition', () => { } }); + it('dispatches existing due nudges before background stale refresh work', async () => { + const claudeRoot = makeTempRoot(); + setClaudeBasePathOverride(claudeRoot); + const teamsBasePath = getTeamsBasePath(); + const teamName = 'team-a'; + const memberName = 'bob'; + let postSeedGetConfigCalls = 0; + let refreshBlocked = false; + let releaseRefresh: () => void = () => undefined; + const refreshBlocker = new Promise((resolve) => { + releaseRefresh = resolve; + }); + const getConfig = vi.fn(async () => { + postSeedGetConfigCalls += 1; + if (postSeedGetConfigCalls === 2) { + refreshBlocked = true; + await refreshBlocker; + } + return { + name: teamName, + members: [{ name: memberName }], + }; + }); + const feature = createMemberWorkSyncFeature({ + teamsBasePath, + configReader: { + getConfig, + } as never, + taskReader: { + getTasks: vi.fn(async () => [ + { + id: 'task-1', + displayId: '11111111', + subject: 'Ship sync', + status: 'pending', + owner: memberName, + }, + ]), + } as never, + kanbanManager: { + getState: vi.fn(async () => ({ + teamName, + reviewers: [], + tasks: {}, + })), + } as never, + membersMetaStore: { + getMembers: vi.fn(async () => []), + } as never, + }); + let dispatchPromise: Promise | null = null; + + try { + await seedShadowReadyMetrics({ teamsBasePath, teamName, memberName }); + const status = await feature.refreshStatus({ teamName, memberName }); + const outboxInput = buildMemberWorkSyncOutboxEnsureInput({ + status, + hash: new NodeHashAdapter(), + nowIso: status.evaluatedAt, + }); + expect(outboxInput).not.toBeNull(); + const store = new JsonMemberWorkSyncStore(new MemberWorkSyncStorePaths(teamsBasePath)); + await expect(store.ensurePending(outboxInput!)).resolves.toMatchObject({ + ok: true, + outcome: 'existing', + }); + + postSeedGetConfigCalls = 0; + dispatchPromise = feature.dispatchDueNudges([teamName]); + await waitForAssertion(() => { + expect(refreshBlocked).toBe(true); + }); + + await expect(readInboxMessages({ teamsBasePath, teamName, memberName })).resolves.toEqual( + expect.arrayContaining([expect.objectContaining({ messageId: outboxInput!.id })]) + ); + + releaseRefresh(); + await expect(dispatchPromise).resolves.toMatchObject({ + claimed: 1, + delivered: 1, + }); + } finally { + releaseRefresh(); + await dispatchPromise?.catch(() => undefined); + await feature.dispose(); + } + }); + it('suppresses queued proof-missing recovery when the original delivery is no longer proof-missing', async () => { const claudeRoot = makeTempRoot(); setClaudeBasePathOverride(claudeRoot); @@ -1115,8 +1204,6 @@ describe('createMemberWorkSyncFeature composition', () => { ); await expect(feature.drainRuntimeTurnSettledEvents()).resolves.toMatchObject({ - claimed: 1, - enqueued: 1, invalid: 0, unresolved: 0, }); @@ -3891,6 +3978,73 @@ describe('createMemberWorkSyncFeature composition', () => { } }); + it('refreshes stale needs_sync into inactive after the whole team stops', async () => { + const claudeRoot = makeTempRoot(); + setClaudeBasePathOverride(claudeRoot); + const teamsBasePath = getTeamsBasePath(); + const teamName = 'team-stopped'; + const memberName = 'bob'; + let teamActive = true; + const feature = createMemberWorkSyncFeature({ + teamsBasePath, + configReader: { + getConfig: vi.fn(async () => ({ + name: teamName, + members: [{ name: memberName, providerId: 'codex' }], + })), + } as never, + taskReader: { + getTasks: vi.fn(async () => [ + { + id: 'task-1', + displayId: '11111111', + subject: 'Finish work after sleep', + status: 'pending', + owner: memberName, + }, + ]), + } as never, + kanbanManager: { + getState: vi.fn(async () => ({ + teamName, + reviewers: [], + tasks: {}, + })), + } as never, + membersMetaStore: { + getMembers: vi.fn(async () => []), + } as never, + isTeamActive: vi.fn(async () => teamActive), + canDispatchNudges: vi.fn(async () => teamActive), + }); + + try { + const current = await feature.refreshStatus({ teamName, memberName }); + expect(current.state).toBe('needs_sync'); + + const store = new JsonMemberWorkSyncStore(new MemberWorkSyncStorePaths(teamsBasePath)); + await store.write({ + ...current, + evaluatedAt: new Date(Date.now() - 3 * 60_000).toISOString(), + }); + teamActive = false; + + await expect(feature.getStatus({ teamName, memberName })).resolves.toMatchObject({ + state: 'needs_sync', + diagnostics: expect.arrayContaining(['status_stale_refresh_enqueued']), + }); + await waitForQueueIdle(feature); + + await expect(store.read({ teamName, memberName })).resolves.toMatchObject({ + state: 'inactive', + diagnostics: expect.arrayContaining(['team_runtime_inactive']), + shadow: { reconciledBy: 'queue', triggerReasons: ['manual_refresh'] }, + }); + } finally { + await feature.dispose(); + } + }); + it('uses snapshot config reads for startup roster materialization', async () => { const getConfig = vi.fn(async () => ({ members: [] })); const getConfigSnapshot = vi.fn(async () => ({ diff --git a/test/main/ipc/configValidation.test.ts b/test/main/ipc/configValidation.test.ts index 442843d4..9c81607e 100644 --- a/test/main/ipc/configValidation.test.ts +++ b/test/main/ipc/configValidation.test.ts @@ -264,6 +264,136 @@ describe('configValidation', () => { } }); + it('accepts Codex custom provider profile updates', () => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: ' http://127.0.0.1:8080/v1 ', + model: ' gateway-codex-model ', + }, + }, + }); + + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.data).toEqual({ + codex: { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'http://127.0.0.1:8080/v1', + model: 'gateway-codex-model', + }, + }, + }); + } + }); + + it('allows disabling Codex custom provider while keeping empty fields', () => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, + }, + }); + + expect(result.valid).toBe(true); + if (result.valid) { + expect(result.data).toEqual({ + codex: { + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, + }, + }); + } + }); + + it.each([ + ['ftp://gateway.example.com/v1', 'http:// or https://'], + ['https://user:token@gateway.example.com/v1', 'credentials'], + ['https://gateway.example.com/v1?token=secret', 'query or fragment'], + ['https://gateway.example.com/v1#token', 'query or fragment'], + ['not a url', 'valid URL'], + ])('rejects invalid Codex custom provider base URL %s', (baseUrl, expectedError) => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + customProvider: { + enabled: true, + baseUrl, + model: 'gateway-codex-model', + }, + }, + }); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toContain(expectedError); + } + }); + + it('requires Codex custom provider model when enabled', () => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: ' ', + }, + }, + }); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toContain('model is required'); + } + }); + + it.each([ + [`gateway\nmodel`, 'control characters'], + ['m'.repeat(201), '200 characters or fewer'], + ])('rejects invalid Codex custom provider model %s', (model, expectedError) => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model, + }, + }, + }); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toContain(expectedError); + } + }); + + it('rejects UI-derived Codex custom provider status fields', () => { + const result = validateConfigUpdatePayload('providerConnections', { + codex: { + customProvider: { + enabled: true, + active: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }, + }); + + expect(result.valid).toBe(false); + if (!result.valid) { + expect(result.error).toContain('active is not a valid setting'); + } + }); + it('accepts Anthropic-compatible endpoint provider connection updates', () => { const result = validateConfigUpdatePayload('providerConnections', { anthropic: { diff --git a/test/main/services/infrastructure/ConfigManager.codexMigration.test.ts b/test/main/services/infrastructure/ConfigManager.codexMigration.test.ts index 16070b20..9883f84a 100644 --- a/test/main/services/infrastructure/ConfigManager.codexMigration.test.ts +++ b/test/main/services/infrastructure/ConfigManager.codexMigration.test.ts @@ -57,11 +57,105 @@ describe('ConfigManager Codex migration hardening', () => { expect(persisted.providerConnections.codex).toEqual({ preferredAuthMode: 'chatgpt', + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, }); expect(persisted.runtime.providerBackends.codex).toBe('codex-native'); }); }); + it('deep-merges and persists Codex custom provider updates', async () => { + tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'config-codex-custom-provider-')); + const configPath = path.join(tempRoot, 'agent-teams-config.json'); + + const { ConfigManager } = await import( + '../../../../src/main/services/infrastructure/ConfigManager' + ); + + const manager = new ConfigManager(configPath); + const updated = manager.updateConfig('providerConnections', { + codex: { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: ' https://gateway.example.com/v1 ', + model: ' gateway-codex-model ', + }, + }, + } as never); + + expect(updated.providerConnections.codex).toEqual({ + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }); + + await vi.waitFor(() => { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- temp fixture path + const persisted = JSON.parse(fs.readFileSync(configPath, 'utf8')) as { + providerConnections: { + codex: { + preferredAuthMode: string; + customProvider: { enabled: boolean; baseUrl: string; model: string }; + }; + }; + }; + + expect(persisted.providerConnections.codex).toEqual({ + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }); + }); + + const disabled = manager.updateConfig('providerConnections', { + codex: { + customProvider: { + enabled: false, + }, + }, + } as never); + + expect(disabled.providerConnections.codex).toEqual({ + preferredAuthMode: 'api_key', + customProvider: { + enabled: false, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }); + + await vi.waitFor(() => { + // eslint-disable-next-line security/detect-non-literal-fs-filename -- temp fixture path + const persisted = JSON.parse(fs.readFileSync(configPath, 'utf8')) as { + providerConnections: { + codex: { + preferredAuthMode: string; + customProvider: { enabled: boolean; baseUrl: string; model: string }; + }; + }; + }; + + expect(persisted.providerConnections.codex).toEqual({ + preferredAuthMode: 'api_key', + customProvider: { + enabled: false, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }); + }); + }); + it('normalizes legacy Codex runtime backend updates inside ConfigManager updateConfig', async () => { tempRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'config-codex-runtime-update-')); const configPath = path.join(tempRoot, 'claude-devtools-config.json'); diff --git a/test/main/services/runtime/ProviderConnectionService.test.ts b/test/main/services/runtime/ProviderConnectionService.test.ts index acbf467c..6cfa124e 100644 --- a/test/main/services/runtime/ProviderConnectionService.test.ts +++ b/test/main/services/runtime/ProviderConnectionService.test.ts @@ -43,7 +43,11 @@ describe('ProviderConnectionService', () => { function createConfig( authMode: 'auto' | 'oauth' | 'api_key' = 'auto', - compatibleEndpoint: { enabled: boolean; baseUrl: string } = { enabled: false, baseUrl: '' } + compatibleEndpoint: { enabled: boolean; baseUrl: string } = { enabled: false, baseUrl: '' }, + codex: Partial<{ + preferredAuthMode: 'auto' | 'chatgpt' | 'api_key'; + customProvider: { enabled: boolean; baseUrl: string; model: string }; + }> = {} ) { return { providerConnections: { @@ -53,7 +57,12 @@ describe('ProviderConnectionService', () => { compatibleEndpoint, }, codex: { - preferredAuthMode: 'auto' as const, + preferredAuthMode: codex.preferredAuthMode ?? ('auto' as const), + customProvider: codex.customProvider ?? { + enabled: false, + baseUrl: '', + model: '', + }, }, }, runtime: { @@ -2180,6 +2189,232 @@ describe('ProviderConnectionService', () => { expect(args).toEqual(['-c', 'forced_login_method="api"']); }); + it('adds custom provider settings for managed Codex API-key launches', async () => { + const { ProviderConnectionService } = + await import('@main/services/runtime/ProviderConnectionService'); + + const service = new ProviderConnectionService( + { + lookupPreferred: vi.fn().mockResolvedValue({ + envVarName: 'OPENAI_API_KEY', + value: 'stored-key', + }), + } as never, + { + getConfig: () => + createConfig('auto', { enabled: false, baseUrl: '' }, { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }), + } as never + ); + + const args = await service.getConfiguredConnectionLaunchArgs( + { + OPENAI_API_KEY: 'stored-key', + CODEX_API_KEY: 'stored-key', + }, + 'codex', + undefined, + '/mock/claude-multimodel' + ); + + expect(args).toEqual([ + '--settings', + JSON.stringify({ + codex: { + forced_login_method: 'api', + agent_teams_custom_provider: { + config_overrides: [ + 'model_provider="agent_teams_custom"', + 'model_providers.agent_teams_custom.name="Agent Teams Custom"', + 'model_providers.agent_teams_custom.base_url="https://gateway.example.com/v1"', + 'model_providers.agent_teams_custom.wire_api="responses"', + 'model_providers.agent_teams_custom.env_key="CODEX_API_KEY"', + ], + }, + }, + }), + ]); + }); + + it('adds direct -c custom provider settings for direct Codex API-key launches', async () => { + const { ProviderConnectionService } = + await import('@main/services/runtime/ProviderConnectionService'); + + const service = new ProviderConnectionService( + { + lookupPreferred: vi.fn().mockResolvedValue({ + envVarName: 'OPENAI_API_KEY', + value: 'stored-key', + }), + } as never, + { + getConfig: () => + createConfig('auto', { enabled: false, baseUrl: '' }, { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'http://127.0.0.1:8080/v1', + model: 'local-codex-model', + }, + }), + } as never + ); + + const args = await service.getConfiguredConnectionLaunchArgs( + { + OPENAI_API_KEY: 'stored-key', + CODEX_API_KEY: 'stored-key', + }, + 'codex', + undefined, + '/usr/local/bin/codex' + ); + + expect(args).toEqual([ + '-c', + 'forced_login_method="api"', + '-c', + 'model_provider="agent_teams_custom"', + '-c', + 'model_providers.agent_teams_custom.name="Agent Teams Custom"', + '-c', + 'model_providers.agent_teams_custom.base_url="http://127.0.0.1:8080/v1"', + '-c', + 'model_providers.agent_teams_custom.wire_api="responses"', + '-c', + 'model_providers.agent_teams_custom.env_key="CODEX_API_KEY"', + ]); + }); + + it('does not pass custom provider settings when Codex resolves to ChatGPT mode', async () => { + const { ProviderConnectionService } = + await import('@main/services/runtime/ProviderConnectionService'); + + const service = new ProviderConnectionService( + { + lookupPreferred: vi.fn().mockResolvedValue({ + envVarName: 'OPENAI_API_KEY', + value: 'stored-key', + }), + } as never, + { + getConfig: () => + createConfig('auto', { enabled: false, baseUrl: '' }, { + preferredAuthMode: 'chatgpt', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }), + } as never + ); + + service.setCodexAccountFeature({ + getSnapshot: vi.fn().mockResolvedValue( + createCodexSnapshot({ + preferredAuthMode: 'chatgpt', + effectiveAuthMode: 'chatgpt', + apiKey: { + available: true, + source: 'stored', + sourceLabel: 'Stored in app', + }, + }) + ), + } as never); + + const args = await service.getConfiguredConnectionLaunchArgs( + { + OPENAI_API_KEY: 'stored-key', + CODEX_API_KEY: 'stored-key', + }, + 'codex', + undefined, + '/mock/claude-multimodel' + ); + + expect(args).toEqual(['--settings', '{"codex":{"forced_login_method":"chatgpt"}}']); + }); + + it('synthesizes the Codex model catalog from the custom provider model', async () => { + const { ProviderConnectionService } = + await import('@main/services/runtime/ProviderConnectionService'); + const directCatalog = vi.fn().mockResolvedValue(null); + + const service = new ProviderConnectionService( + { + lookupPreferred: vi.fn().mockResolvedValue({ + envVarName: 'OPENAI_API_KEY', + value: 'stored-key', + }), + } as never, + { + getConfig: () => + createConfig('auto', { enabled: false, baseUrl: '' }, { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }), + } as never + ); + service.setCodexModelCatalogFeature({ getCatalog: directCatalog } as never); + + const enriched = await service.enrichProviderStatus({ + providerId: 'codex', + displayName: 'Codex', + supported: true, + authenticated: true, + authMethod: 'api_key', + verificationState: 'verified', + models: ['gpt-5.4'], + subscriptionRateLimits: { + primary: null, + secondary: null, + }, + runtimeCapabilities: { + modelCatalog: { dynamic: true, source: 'app-server' }, + }, + canLoginFromUi: false, + capabilities: { + teamLaunch: true, + oneShot: true, + extensions: { + plugins: { status: 'unsupported', ownership: 'shared' }, + mcp: { status: 'supported', ownership: 'shared' }, + skills: { status: 'supported', ownership: 'shared' }, + apiKeys: { status: 'supported', ownership: 'shared' }, + }, + }, + }); + + expect(directCatalog).not.toHaveBeenCalled(); + expect(enriched.models).toEqual(['gateway-codex-model']); + expect(enriched.modelCatalog?.defaultLaunchModel).toBe('gateway-codex-model'); + expect(enriched.modelCatalog?.models).toHaveLength(1); + expect(enriched.modelCatalog?.models[0]).toMatchObject({ + id: 'gateway-codex-model', + launchModel: 'gateway-codex-model', + supportsFastMode: false, + source: 'static-fallback', + }); + expect(enriched.subscriptionRateLimits).toBeNull(); + expect(enriched.backend?.endpointLabel).toBe('https://gateway.example.com/v1'); + expect(enriched.runtimeCapabilities?.modelCatalog).toEqual({ + dynamic: false, + source: 'static-fallback', + }); + }); + it('prefers the orchestrator Codex model catalog over the legacy direct app-server fallback', async () => { const { ProviderConnectionService } = await import('@main/services/runtime/ProviderConnectionService'); diff --git a/test/main/services/runtime/providerAwareCliEnv.test.ts b/test/main/services/runtime/providerAwareCliEnv.test.ts index 7d4c3265..fdc4d17c 100644 --- a/test/main/services/runtime/providerAwareCliEnv.test.ts +++ b/test/main/services/runtime/providerAwareCliEnv.test.ts @@ -454,6 +454,49 @@ describe('buildProviderAwareCliEnv', () => { ]); }); + it('returns Codex custom provider launch args after API-key env application', async () => { + applyConfiguredConnectionEnvMock.mockImplementation(async (env: NodeJS.ProcessEnv) => { + env.OPENAI_API_KEY = 'stored-key'; + env.CODEX_API_KEY = 'stored-key'; + return env; + }); + const customSettings = JSON.stringify({ + codex: { + forced_login_method: 'api', + agent_teams_custom_provider: { + config_overrides: [ + 'model_provider="agent_teams_custom"', + 'model_providers.agent_teams_custom.name="Agent Teams Custom"', + 'model_providers.agent_teams_custom.base_url="https://gateway.example.com/v1"', + 'model_providers.agent_teams_custom.wire_api="responses"', + 'model_providers.agent_teams_custom.env_key="CODEX_API_KEY"', + ], + }, + }, + }); + getConfiguredConnectionLaunchArgsMock.mockResolvedValue(['--settings', customSettings]); + + const { buildProviderAwareCliEnv } = + await import('../../../../src/main/services/runtime/providerAwareCliEnv'); + const result = await buildProviderAwareCliEnv({ + binaryPath: '/mock/claude-multimodel', + providerId: 'codex', + }); + + expect(getConfiguredConnectionLaunchArgsMock).toHaveBeenCalledWith( + expect.objectContaining({ + OPENAI_API_KEY: 'stored-key', + CODEX_API_KEY: 'stored-key', + }), + 'codex', + undefined, + '/mock/claude-multimodel' + ); + expect(result.providerArgs).toEqual(['--settings', customSettings]); + expect(result.env.OPENAI_API_KEY).toBe('stored-key'); + expect(result.env.CODEX_API_KEY).toBe('stored-key'); + }); + it('passes Codex env refreshed by strict credential application into launch args and issue checks', async () => { applyConfiguredConnectionEnvMock.mockImplementation( async (env: NodeJS.ProcessEnv, providerId: string) => { diff --git a/test/main/services/runtime/providerModelProbe.test.ts b/test/main/services/runtime/providerModelProbe.test.ts new file mode 100644 index 00000000..115dfded --- /dev/null +++ b/test/main/services/runtime/providerModelProbe.test.ts @@ -0,0 +1,22 @@ +import { + buildProviderPreflightPingArgs, + getProviderPreflightModel, +} from '@main/services/runtime/providerModelProbe'; +import { describe, expect, it } from 'vitest'; + +describe('providerModelProbe', () => { + it('uses the configured model override for Codex preflight probes', () => { + expect(getProviderPreflightModel('codex', { modelOverride: 'gateway-codex-model' })).toBe( + 'gateway-codex-model' + ); + + expect( + buildProviderPreflightPingArgs('codex', { modelOverride: 'gateway-codex-model' }) + ).toContain('gateway-codex-model'); + }); + + it('keeps the default Codex preflight model when no override is configured', () => { + expect(getProviderPreflightModel('codex')).toBe('gpt-5.4-mini'); + expect(buildProviderPreflightPingArgs('codex')).toContain('gpt-5.4-mini'); + }); +}); diff --git a/test/main/services/team/ProcessBootstrapTransportEvidence.test.ts b/test/main/services/team/ProcessBootstrapTransportEvidence.test.ts index c247ff2c..6daf309c 100644 --- a/test/main/services/team/ProcessBootstrapTransportEvidence.test.ts +++ b/test/main/services/team/ProcessBootstrapTransportEvidence.test.ts @@ -96,6 +96,30 @@ describe('ProcessBootstrapTransportEvidence', () => { expect(summary?.lastStage).toBe('process spawned'); }); + it('surfaces headless startup checkpoints as transport progress', () => { + const summary = summarizeProcessBootstrapTransportEvents([ + { + type: 'cli_started', + timestamp: '2026-05-07T10:00:00.000Z', + detail: 'teammateRuntime=headless', + }, + { + type: 'startup_checkpoint', + timestamp: '2026-05-07T10:00:01.000Z', + detail: 'commands_agents_loaded', + }, + ]); + + expect(summary).toMatchObject({ + submitted: false, + hasProgress: true, + lastStage: 'startup checkpoint: commands_agents_loaded', + }); + expect(buildProcessBootstrapTimeoutDiagnostic(summary!)).toBe( + 'Bootstrap prompt was not submitted before timeout. Last transport stage: startup checkpoint: commands_agents_loaded' + ); + }); + it('builds stable pending and timeout diagnostics from the last transport stage', () => { const summary = summarizeProcessBootstrapTransportEvents([ { diff --git a/test/main/services/team/TeamLaunchFailureArtifactPack.test.ts b/test/main/services/team/TeamLaunchFailureArtifactPack.test.ts index 620e04c8..f2cd1c63 100644 --- a/test/main/services/team/TeamLaunchFailureArtifactPack.test.ts +++ b/test/main/services/team/TeamLaunchFailureArtifactPack.test.ts @@ -276,6 +276,26 @@ describe('TeamLaunchFailureArtifactPack', () => { }); }); + it('extracts startup checkpoint runtime stages and keeps stdin warning secondary', () => { + const input = { + teamName: 'artifact-team', + runId: 'run-startup-checkpoint', + reason: + 'alice: Teammate process alice@signal-ops did not become runtime_ready: timed out waiting for runtime_ready; last runtime stage: startup_checkpoint: commands_agents_loaded Last stderr: Warning: no stdin data received in 3s, proceeding without it.', + progressTraceLines: [ + 'startup_checkpoint detail=commands_agents_loaded', + 'Warning: no stdin data received in 3s, proceeding without it.', + ], + }; + + expect(classifyLaunchFailureArtifact(input).code).toBe('process_readiness_timeout'); + expect(extractLaunchBootstrapTransportBreadcrumb(input)).toMatchObject({ + lastTransportStage: 'startup_checkpoint: commands_agents_loaded', + noStdinWarning: true, + bootstrapSubmitted: false, + }); + }); + it('keeps inbox poller bootstrap stalls out of stdin_missing classification', () => { const input = { teamName: 'artifact-team', diff --git a/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts b/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts index 99a94de7..cb705220 100644 --- a/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts +++ b/test/main/services/team/TeamProvisioningLaunchFailurePolicy.test.ts @@ -3,8 +3,8 @@ import { isAutoClearableLaunchFailureReason, isBootstrapCheckInTimeoutFailureReason, isBootstrapInstructionPromptFailureReason, - isCliProvisionedButNotAliveFailureReason, isBootstrapMcpResourceReadFailureReason, + isCliProvisionedButNotAliveFailureReason, isConfigRegistrationFailureReason, isLaunchCleanupBootstrapIncompleteFailureReason, isLaunchGraceWindowFailureReason, @@ -68,6 +68,16 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'Teammate was registered but did not bootstrap-confirm before timeout.' ) ).toBe(true); + expect( + isBootstrapCheckInTimeoutFailureReason( + 'Bootstrap prompt was submitted, but teammate did not bootstrap-confirm before submitted-confirmation timeout (3m). Last transport stage: bootstrap_submitted' + ) + ).toBe(true); + expect( + isBootstrapCheckInTimeoutFailureReason( + 'Bootstrap prompt was submitted, but teammate did not bootstrap-confirm before timeout.' + ) + ).toBe(false); expect( isBootstrapInstructionPromptFailureReason( 'You are bootstrapping into team atlas. Your first action is to call the MCP tool member_briefing.' @@ -107,6 +117,16 @@ describe('TeamProvisioningLaunchFailurePolicy', () => { 'Teammate did not join within the launch grace window.; process table unavailable' ) ).toBe(true); + expect( + isAutoClearableLaunchFailureReason( + 'Bootstrap prompt was submitted, but teammate did not bootstrap-confirm before submitted-confirmation timeout (3m). Last transport stage: bootstrap_submitted' + ) + ).toBe(true); + expect( + isAutoClearableLaunchFailureReason( + 'Bootstrap prompt was submitted, but teammate did not bootstrap-confirm before timeout.' + ) + ).toBe(false); expect( isAutoClearableLaunchFailureReason( 'CLI process exited (code 1) \u2014 team provisioned but not alive' diff --git a/test/main/services/team/TeamProvisioningService.test.ts b/test/main/services/team/TeamProvisioningService.test.ts index 86609500..85381cd7 100644 --- a/test/main/services/team/TeamProvisioningService.test.ts +++ b/test/main/services/team/TeamProvisioningService.test.ts @@ -711,9 +711,7 @@ type TeamProvisioningServicePrivateHarness = { applyProcessBootstrapTransportOverlay: ( input: Record ) => Record; - reconcilePersistedLaunchState: ( - teamName: string - ) => Promise<{ + reconcilePersistedLaunchState: (teamName: string) => Promise<{ snapshot: null; statuses: Record; }>; @@ -1152,6 +1150,104 @@ describe('TeamProvisioningService', () => { expect(nextRecord.status).toBe('retry_scheduled'); }); + it('emits a terminal failure event when exhausted work-sync proof retries fail', async () => { + const svc = new TeamProvisioningService(); + const taskRefs = [{ taskId: 'task-1', displayId: 'task-1', teamName: 'team-a' }]; + const record = { + id: 'opencode-prompt:work-sync-proof-missing', + teamName: 'team-a', + memberName: 'atlas', + laneId: 'secondary:opencode:atlas', + runId: 'run-1', + runtimeSessionId: 'ses-1', + inboxMessageId: 'msg-work-sync-proof-missing', + inboxTimestamp: '2026-05-18T08:31:00.000Z', + source: 'watcher', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + replyRecipient: 'team-lead', + actionMode: 'do', + taskRefs, + payloadHash: 'sha256:work-sync', + status: 'retry_scheduled', + responseState: 'responded_non_visible_tool', + attempts: 3, + maxAttempts: 3, + acceptanceUnknown: false, + nextAttemptAt: null, + lastAttemptAt: '2026-05-18T08:31:30.000Z', + lastObservedAt: '2026-05-18T08:31:45.000Z', + acceptedAt: '2026-05-18T08:31:30.000Z', + respondedAt: '2026-05-18T08:31:45.000Z', + failedAt: null, + inboxReadCommittedAt: null, + inboxReadCommitError: null, + prePromptCursor: null, + postPromptCursor: null, + deliveredUserMessageId: 'delivered-1', + observedAssistantMessageId: 'assistant-1', + observedAssistantPreview: null, + observedToolCallNames: ['member_work_sync_status'], + observedVisibleMessageId: null, + visibleReplyMessageId: null, + visibleReplyInbox: null, + visibleReplyCorrelation: null, + lastReason: 'member_work_sync_report_required', + diagnostics: ['member_work_sync_report_required'], + createdAt: '2026-05-18T08:31:00.000Z', + updatedAt: '2026-05-18T08:31:45.000Z', + }; + const failedRecord = { + ...record, + status: 'failed_terminal', + failedAt: '2026-05-18T08:32:00.000Z', + updatedAt: '2026-05-18T08:32:00.000Z', + }; + const ledger = { + markFailedTerminal: vi.fn(async () => failedRecord), + markNextAttemptScheduled: vi.fn(), + }; + const harness = svc as unknown as { + scheduleOpenCodePromptDeliveryWatchdog: ReturnType; + logOpenCodePromptDeliveryEvent: ReturnType; + scheduleOpenCodePromptLedgerFollowUp(input: { + ledger: typeof ledger; + ledgerRecord: typeof record; + teamName: string; + memberName: string; + retry: boolean; + reason: string; + }): Promise; + }; + harness.scheduleOpenCodePromptDeliveryWatchdog = vi.fn(); + harness.logOpenCodePromptDeliveryEvent = vi.fn(); + + const nextRecord = await harness.scheduleOpenCodePromptLedgerFollowUp({ + ledger, + ledgerRecord: record, + teamName: 'team-a', + memberName: 'atlas', + retry: true, + reason: 'member_work_sync_report_required', + }); + + expect(nextRecord).toBe(failedRecord); + expect(ledger.markFailedTerminal).toHaveBeenCalledWith( + expect.objectContaining({ + id: record.id, + reason: 'member_work_sync_report_required', + }) + ); + expect(harness.logOpenCodePromptDeliveryEvent).toHaveBeenCalledWith( + 'opencode_prompt_delivery_terminal_failure', + failedRecord, + expect.objectContaining({ + reason: 'member_work_sync_report_required', + retry: true, + }) + ); + }); + it('uses stamped OpenCode session-refresh evidence instead of stale historical diagnostics', async () => { const svc = new TeamProvisioningService(); (svc as any).scheduleOpenCodePromptDeliveryWatchdog = vi.fn(); @@ -16725,8 +16821,7 @@ describe('TeamProvisioningService', () => { return launchIdentity; }); (svc as any).buildTeamRuntimeLaunchArgsPlan = vi.fn(async (input) => ({ - fastModeArgs: - input.launchIdentity === launchIdentity ? ['--test-codex-fast-mode'] : [], + fastModeArgs: input.launchIdentity === launchIdentity ? ['--test-codex-fast-mode'] : [], runtimeTurnSettledHookArgs: [], providerArgs: [], settingsArgs: [], @@ -21326,7 +21421,8 @@ describe('TeamProvisioningService', () => { status: 'failed', lastAttemptAt: Date.parse(acceptedAt), lastObservedAt: Date.parse(failureAt), - failureReason: 'Teammate was registered but did not bootstrap-confirm before timeout.', + failureReason: + 'Bootstrap prompt was submitted, but teammate did not bootstrap-confirm before submitted-confirmation timeout (3m). Last transport stage: bootstrap_submitted', }, ], failureAt @@ -22260,9 +22356,10 @@ describe('TeamProvisioningService', () => { expect(bobOutcome).toBeNull(); // The transcript tail is parsed once and shared: a single cache entry for the // file rather than one parse per member. - expect((svc as unknown as Record>).parsedBootstrapTranscriptTailCache.size).toBe( - 1 - ); + expect( + (svc as unknown as Record>).parsedBootstrapTranscriptTailCache + .size + ).toBe(1); }); it('caches persisted bootstrap transcript outcome lookup between close polling reads', async () => { @@ -24523,12 +24620,10 @@ describe('TeamProvisioningService', () => { scheduled: true, reason: 'scheduled', })); - const sendMessageToRun = vi.fn( - async (targetRun: LeadRelayPriorityTestRun, message: string) => { - deliveredPrompt = message; - targetRun.leadRelayCapture?.resolveOnce(''); - } - ); + const sendMessageToRun = vi.fn(async (targetRun: LeadRelayPriorityTestRun, message: string) => { + deliveredPrompt = message; + targetRun.leadRelayCapture?.resolveOnce(''); + }); harness.runs.set(run.runId, run); harness.aliveRunByTeam.set(teamName, run.runId); @@ -25854,23 +25949,22 @@ describe('TeamProvisioningService', () => { it('does not keep healed confirmed-bootstrap status alive when refreshed runtime metadata is an error', async () => { const svc = new TeamProvisioningService(); const harness = privateHarness(svc); - harness.getLiveTeamAgentRuntimeMetadata = vi.fn( - () => - Promise.resolve( - new Map([ - [ - 'tom', - { - alive: false, - model: 'sonnet', - livenessKind: 'not_found', - pidSource: 'process_table', - runtimeDiagnostic: 'Runtime process crashed', - runtimeDiagnosticSeverity: 'error', - }, - ], - ]) - ) + harness.getLiveTeamAgentRuntimeMetadata = vi.fn(() => + Promise.resolve( + new Map([ + [ + 'tom', + { + alive: false, + model: 'sonnet', + livenessKind: 'not_found', + pidSource: 'process_table', + runtimeDiagnostic: 'Runtime process crashed', + runtimeDiagnosticSeverity: 'error', + }, + ], + ]) + ) ); const result = await harness.attachLiveRuntimeMetadataToStatuses('signal-ops', { diff --git a/test/main/services/team/TeamProvisioningServiceRelay.test.ts b/test/main/services/team/TeamProvisioningServiceRelay.test.ts index 288bc228..bd68f65f 100644 --- a/test/main/services/team/TeamProvisioningServiceRelay.test.ts +++ b/test/main/services/team/TeamProvisioningServiceRelay.test.ts @@ -1630,6 +1630,81 @@ Messages: expect(payload).toContain('Please retry with logging enabled.'); }); + it('prioritizes member work-sync nudges over older ordinary member relay rows', async () => { + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + seedConfig(teamName); + seedMemberInbox(teamName, 'alice', [ + ...Array.from({ length: 11 }, (_, index) => ({ + from: 'team-lead', + text: `Routine relay row ${index + 1}.`, + timestamp: `2026-02-23T10:${String(index).padStart(2, '0')}:00.000Z`, + read: false, + messageId: `m-ordinary-${index + 1}`, + })), + { + from: 'system', + text: 'Call member_work_sync_status, then member_work_sync_report.', + timestamp: '2026-02-23T10:30:00.000Z', + read: false, + messageId: 'm-work-sync-late', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + }, + ]); + + const { writeSpy } = attachAliveRun(service, teamName); + const relayed = await service.relayMemberInboxMessages(teamName, 'alice'); + + expect(relayed).toBe(10); + const payload = String(writeSpy.mock.calls[0]?.[0] ?? ''); + expect(payload).toContain('1) From: system'); + expect(payload).toContain('MessageId: m-work-sync-late'); + expect(payload).toContain('Message kind: member_work_sync_nudge'); + expect(payload).not.toContain('MessageId: m-ordinary-11'); + }); + + it('retries a work-sync nudge after member relay times out before stdin write completes', async () => { + vi.useFakeTimers(); + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + try { + seedConfig(teamName); + seedMemberInbox(teamName, 'alice', [ + { + from: 'system', + text: 'Call member_work_sync_status, then member_work_sync_report.', + timestamp: '2026-02-23T10:00:00.000Z', + read: false, + messageId: 'm-work-sync-retry', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + }, + ]); + + const { writeSpy } = attachAliveRun(service, teamName); + writeSpy.mockImplementationOnce(() => true); + + const firstRelay = service.relayMemberInboxMessages(teamName, 'alice'); + await vi.advanceTimersByTimeAsync(0); + expect(writeSpy).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(120_000); + await expect(firstRelay).resolves.toBe(0); + vi.mocked(console.warn).mockClear(); + + const secondRelay = await service.relayMemberInboxMessages(teamName, 'alice'); + + expect(secondRelay).toBe(1); + expect(writeSpy).toHaveBeenCalledTimes(2); + const secondPayload = String(writeSpy.mock.calls[1]?.[0] ?? ''); + expect(secondPayload).toContain('MessageId: m-work-sync-retry'); + expect(secondPayload).toContain('Message kind: member_work_sync_nudge'); + } finally { + vi.useRealTimers(); + } + }); + it('marks exact teammate relay copies with relayOfMessageId', async () => { const service = new TeamProvisioningService(); const teamName = 'my-team'; @@ -3445,6 +3520,9 @@ Messages: }, ]); const deliverSpy = vi.spyOn(service, 'deliverOpenCodeMemberMessage'); + const logSpy = vi + .spyOn(service as any, 'logOpenCodePromptDeliveryEvent') + .mockImplementation(() => undefined); const relay = await service.relayOpenCodeMemberInboxMessages(teamName, 'jack'); const expectedReason = 'opencode_inbox_attachment_payload_unavailable: att-1'; @@ -3469,6 +3547,18 @@ Messages: status: 'failed_terminal', lastReason: expectedReason, }); + expect(logSpy).toHaveBeenCalledWith( + 'opencode_prompt_delivery_terminal_failure', + expect.objectContaining({ + inboxMessageId: 'opencode-attachment-1', + status: 'failed_terminal', + lastReason: expectedReason, + }), + expect.objectContaining({ + attachmentPayloadUnavailable: true, + reason: expectedReason, + }) + ); }); it('rebuilds missing OpenCode prompt ledger rows from unread inbox on startup scan', async () => { @@ -3721,6 +3811,224 @@ Messages: } }); + it('keeps an already-read work-sync nudge pending when it is queued behind an active relay', async () => { + vi.useFakeTimers(); + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + try { + hoisted.files.set( + `/mock/teams/${teamName}/config.json`, + JSON.stringify({ + name: teamName, + projectPath: '/tmp/my-team', + members: [ + { name: 'team-lead', agentType: 'team-lead' }, + { name: 'jack', role: 'developer', providerId: 'opencode', model: 'openrouter/test' }, + ], + }) + ); + seedMemberInbox(teamName, 'jack', [ + { + from: 'bob', + to: 'jack', + text: 'Older watcher message.', + timestamp: '2026-02-23T17:00:00.000Z', + read: false, + messageId: 'opencode-inflight-old', + }, + ]); + + const oldDeliveryStarted = createDeferred(); + const releaseOldDelivery = createDeferred(); + vi.spyOn(service, 'deliverOpenCodeMemberMessage').mockImplementation( + async (_teamName, input) => { + if (input.messageId === 'opencode-inflight-old') { + oldDeliveryStarted.resolve(undefined); + await releaseOldDelivery.promise; + } + return { delivered: true, diagnostics: [] }; + } + ); + const wakeSpy = vi + .spyOn(service, 'scheduleOpenCodeMemberInboxDeliveryWake') + .mockImplementation(() => undefined); + + const watcherRelay = service.relayOpenCodeMemberInboxMessages(teamName, 'jack'); + await oldDeliveryStarted.promise; + seedMemberInbox(teamName, 'jack', [ + { + from: 'bob', + to: 'jack', + text: 'Older watcher message.', + timestamp: '2026-02-23T17:00:00.000Z', + read: false, + messageId: 'opencode-inflight-old', + }, + { + from: 'system', + to: 'jack', + text: 'Call member_work_sync_status, then member_work_sync_report.', + timestamp: '2026-02-23T17:00:01.000Z', + read: true, + messageId: 'work-sync-read-queued', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + }, + ]); + + await expect( + service.relayOpenCodeMemberInboxMessages(teamName, 'jack', { + onlyMessageId: 'work-sync-read-queued', + source: 'watchdog', + }) + ).resolves.toMatchObject({ + attempted: 1, + delivered: 0, + failed: 0, + lastDelivery: { + delivered: true, + accepted: false, + responsePending: true, + reason: 'opencode_work_sync_read_commit_waiting_for_active_relay', + }, + }); + expect(wakeSpy).toHaveBeenCalledWith({ + teamName, + memberName: 'jack', + messageId: 'work-sync-read-queued', + delayMs: 500, + }); + + releaseOldDelivery.resolve(undefined); + await watcherRelay; + } finally { + vi.useRealTimers(); + } + }); + + it('times out a hung existing OpenCode member relay in-flight lock', async () => { + vi.useFakeTimers(); + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + const relayKey = `opencode:${teamName}:jack`; + try { + ( + service as unknown as { + openCodeMemberInboxRelayInFlight: Map>; + } + ).openCodeMemberInboxRelayInFlight.set(relayKey, new Promise(() => undefined)); + + const relay = service.relayOpenCodeMemberInboxMessages(teamName, 'jack'); + await vi.advanceTimersByTimeAsync(120_000); + + await expect(relay).resolves.toMatchObject({ + attempted: 0, + delivered: 0, + failed: 1, + lastDelivery: { + delivered: false, + accepted: false, + responsePending: false, + reason: 'opencode_member_inbox_relay_timed_out', + }, + }); + expect( + ( + service as unknown as { + openCodeMemberInboxRelayInFlight: Map>; + } + ).openCodeMemberInboxRelayInFlight.has(relayKey) + ).toBe(false); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'opencode_member_inbox_relay_timed_out' + ); + vi.mocked(console.warn).mockClear(); + } finally { + vi.useRealTimers(); + } + }); + + it('times out a hung existing lead relay in-flight lock', async () => { + vi.useFakeTimers(); + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + try { + ( + service as unknown as { + leadInboxRelayInFlight: Map>; + } + ).leadInboxRelayInFlight.set(teamName, new Promise(() => undefined)); + + const relay = service.relayLeadInboxMessages(teamName); + await vi.advanceTimersByTimeAsync(120_000); + + await expect(relay).resolves.toBe(0); + expect( + ( + service as unknown as { + leadInboxRelayInFlight: Map>; + } + ).leadInboxRelayInFlight.has(teamName) + ).toBe(false); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'lead_inbox_relay_timed_out' + ); + vi.mocked(console.warn).mockClear(); + } finally { + vi.useRealTimers(); + } + }); + + it('times out a hung existing member relay in-flight lock', async () => { + vi.useFakeTimers(); + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + const relayKey = `${teamName}:alice`; + try { + ( + service as unknown as { + memberInboxRelayInFlight: Map>; + } + ).memberInboxRelayInFlight.set(relayKey, new Promise(() => undefined)); + + const relay = service.relayMemberInboxMessages(teamName, 'alice'); + await vi.advanceTimersByTimeAsync(120_000); + + await expect(relay).resolves.toBe(0); + expect( + ( + service as unknown as { + memberInboxRelayInFlight: Map>; + } + ).memberInboxRelayInFlight.has(relayKey) + ).toBe(false); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'member_inbox_relay_timed_out' + ); + vi.mocked(console.warn).mockClear(); + } finally { + vi.useRealTimers(); + } + }); + + it('does not convert non-timeout member relay failures into timeout results', async () => { + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + const relayKey = `${teamName}:alice`; + const rejected = Promise.reject(new Error('relay failed')); + rejected.catch(() => undefined); + ( + service as unknown as { + memberInboxRelayInFlight: Map>; + } + ).memberInboxRelayInFlight.set(relayKey, rejected); + + await expect(service.relayMemberInboxMessages(teamName, 'alice')).rejects.toThrow( + 'relay failed' + ); + expect(vi.mocked(console.warn)).not.toHaveBeenCalled(); + }); + it('treats an already-read specific OpenCode inbox row as delivered for UI-send relay', async () => { const service = new TeamProvisioningService(); const teamName = 'my-team'; @@ -3762,6 +4070,68 @@ Messages: expect(deliverSpy).not.toHaveBeenCalled(); }); + it('does not treat an already-read work-sync nudge as delivered without the work-sync proof path', async () => { + const service = new TeamProvisioningService(); + const teamName = 'my-team'; + hoisted.files.set( + `/mock/teams/${teamName}/config.json`, + JSON.stringify({ + name: teamName, + projectPath: '/tmp/my-team', + members: [ + { name: 'team-lead', agentType: 'team-lead' }, + { name: 'jack', role: 'developer', providerId: 'opencode', model: 'openrouter/test' }, + ], + }) + ); + seedMemberInbox(teamName, 'jack', [ + { + from: 'system', + to: 'jack', + text: 'Call member_work_sync_status, then member_work_sync_report.', + timestamp: '2026-02-23T17:02:00.000Z', + read: true, + messageId: 'work-sync-read-1', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + taskRefs: [{ taskId: 'task-1', teamName }], + }, + ]); + const deliverSpy = vi.spyOn(service, 'deliverOpenCodeMemberMessage').mockResolvedValue({ + delivered: true, + accepted: false, + responsePending: true, + reason: 'member_work_sync_report_required', + diagnostics: ['member_work_sync_report_required'], + }); + + const relay = await service.relayOpenCodeMemberInboxMessages(teamName, 'jack', { + onlyMessageId: 'work-sync-read-1', + source: 'watchdog', + }); + + expect(deliverSpy).toHaveBeenCalledWith( + teamName, + expect.objectContaining({ + memberName: 'jack', + messageId: 'work-sync-read-1', + messageKind: 'member_work_sync_nudge', + workSyncIntent: 'agenda_sync', + }) + ); + expect(relay).toMatchObject({ + attempted: 1, + delivered: 0, + failed: 0, + lastDelivery: { + delivered: true, + accepted: false, + responsePending: true, + reason: 'member_work_sync_report_required', + }, + }); + }); + it('routes watcher inbox changes for OpenCode members through direct runtime relay', async () => { const service = new TeamProvisioningService(); const teamName = 'my-team'; @@ -4357,7 +4727,10 @@ Messages: ], }) ); - hoisted.files.set(`${teamsBasePath}/${teamName}/inboxes/${memberName}.json`, JSON.stringify([])); + hoisted.files.set( + `${teamsBasePath}/${teamName}/inboxes/${memberName}.json`, + JSON.stringify([]) + ); (service as any).resolveOpenCodeMemberDeliveryIdentity = vi.fn(async () => ({ ok: true, canonicalMemberName: memberName, @@ -4415,7 +4788,10 @@ Messages: ], }) ); - hoisted.files.set(`${teamsBasePath}/${teamName}/inboxes/${memberName}.json`, JSON.stringify([])); + hoisted.files.set( + `${teamsBasePath}/${teamName}/inboxes/${memberName}.json`, + JSON.stringify([]) + ); (service as any).resolveOpenCodeMemberDeliveryIdentity = vi.fn(async () => ({ ok: true, canonicalMemberName: memberName, diff --git a/test/main/services/team/TeamReconcileDrainScheduler.test.ts b/test/main/services/team/TeamReconcileDrainScheduler.test.ts index 98efd778..c8976ce8 100644 --- a/test/main/services/team/TeamReconcileDrainScheduler.test.ts +++ b/test/main/services/team/TeamReconcileDrainScheduler.test.ts @@ -30,13 +30,14 @@ function createDeferred(): Deferred { } async function flushAsyncWork(): Promise { - await Promise.resolve(); - await Promise.resolve(); - await Promise.resolve(); + for (let i = 0; i < 8; i += 1) { + await Promise.resolve(); + } } describe('TeamReconcileDrainScheduler', () => { afterEach(() => { + vi.useRealTimers(); vi.restoreAllMocks(); mockYieldToEventLoop.mockReset(); }); @@ -176,6 +177,72 @@ describe('TeamReconcileDrainScheduler', () => { scheduler.dispose(); }); + it('times out a hung run so pending team reconciles can continue', async () => { + vi.useFakeTimers(); + mockYieldToEventLoop.mockResolvedValue(undefined); + const hungRun = createDeferred(); + const run = vi + .fn<(teamName: string, trigger: TeamReconcileTrigger) => Promise>() + .mockImplementationOnce(async () => { + await hungRun.promise; + }) + .mockResolvedValueOnce(undefined); + const scheduler = createTeamReconcileDrainScheduler({ + run, + runTimeoutMs: 10, + }); + + scheduler.schedule('team-a', { source: 'inbox', detail: 'inboxes/alice.json' }); + await flushAsyncWork(); + expect(run).toHaveBeenCalledTimes(1); + + scheduler.schedule('team-a', { source: 'task', detail: 'task-2.json' }); + await flushAsyncWork(); + expect(run).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(10); + await flushAsyncWork(); + + expect(run).toHaveBeenCalledTimes(2); + expect(run).toHaveBeenNthCalledWith(2, 'team-a', { + source: 'task', + detail: 'task-2.json', + }); + + scheduler.dispose(); + }); + + it('retries the timed out trigger when no newer event arrived', async () => { + vi.useFakeTimers(); + mockYieldToEventLoop.mockResolvedValue(undefined); + const hungRun = createDeferred(); + const run = vi + .fn<(teamName: string, trigger: TeamReconcileTrigger) => Promise>() + .mockImplementationOnce(async () => { + await hungRun.promise; + }) + .mockResolvedValueOnce(undefined); + const scheduler = createTeamReconcileDrainScheduler({ + run, + runTimeoutMs: 10, + }); + + scheduler.schedule('team-a', { source: 'inbox', detail: 'inboxes/alice.json' }); + await flushAsyncWork(); + expect(run).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(10); + await flushAsyncWork(); + + expect(run).toHaveBeenCalledTimes(2); + expect(run).toHaveBeenNthCalledWith(2, 'team-a', { + source: 'inbox', + detail: 'inboxes/alice.json', + }); + + scheduler.dispose(); + }); + it('does not lose a new event that arrives while a failed pass is yielding', async () => { const yieldGate = createDeferred(); mockYieldToEventLoop.mockImplementationOnce(() => yieldGate.promise).mockResolvedValue(undefined); diff --git a/test/main/services/team/stallMonitor/ActiveTeamRegistry.test.ts b/test/main/services/team/stallMonitor/ActiveTeamRegistry.test.ts index e27a7c6f..8eef0032 100644 --- a/test/main/services/team/stallMonitor/ActiveTeamRegistry.test.ts +++ b/test/main/services/team/stallMonitor/ActiveTeamRegistry.test.ts @@ -3,6 +3,20 @@ import { describe, expect, it, vi } from 'vitest'; import { ActiveTeamRegistry } from '../../../../../src/main/services/team/stallMonitor/ActiveTeamRegistry'; describe('ActiveTeamRegistry', () => { + function createDeferred(): { + promise: Promise; + resolve: (value: T) => void; + reject: (error: unknown) => void; + } { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((promiseResolve, promiseReject) => { + resolve = promiseResolve; + reject = promiseReject; + }); + return { promise, resolve, reject }; + } + it('activates a team on lead-activity and enables stall-monitor tracking', async () => { const tracker = { enableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), @@ -99,6 +113,92 @@ describe('ActiveTeamRegistry', () => { await expect(registry.listActiveTeams()).resolves.toEqual(['beta']); }); + it('retries activation when enabling stall-monitor tracking fails', async () => { + const tracker = { + enableTracking: vi + .fn() + .mockRejectedValueOnce(new Error('tracker unavailable')) + .mockResolvedValueOnce({ projectFingerprint: null, logSourceGeneration: null }), + disableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), + }; + const registry = new ActiveTeamRegistry( + { listAliveProcessTeams: vi.fn(async () => ['demo']) }, + tracker as never + ); + + registry.noteTeamChange({ + type: 'lead-activity', + teamName: 'demo', + detail: 'active', + }); + + await vi.waitFor(() => { + expect(tracker.enableTracking).toHaveBeenCalledTimes(1); + }); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'Failed to enable stall-monitor tracking for demo' + ); + vi.mocked(console.warn).mockClear(); + await expect(registry.listActiveTeams()).resolves.toEqual([]); + + await registry.reconcile(); + + expect(tracker.enableTracking).toHaveBeenCalledTimes(2); + await expect(registry.listActiveTeams()).resolves.toEqual(['demo']); + }); + + it('does not re-add a team when pending activation finishes after stop', async () => { + const activation = createDeferred<{ + projectFingerprint: string | null; + logSourceGeneration: string | null; + }>(); + const tracker = { + enableTracking: vi.fn(() => activation.promise), + disableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), + }; + const registry = new ActiveTeamRegistry( + { listAliveProcessTeams: vi.fn(async () => []) }, + tracker as never + ); + + registry.noteTeamChange({ + type: 'lead-activity', + teamName: 'demo', + detail: 'active', + }); + await vi.waitFor(() => { + expect(tracker.enableTracking).toHaveBeenCalledWith('demo', 'stall_monitor'); + }); + + await registry.stop(); + activation.resolve({ projectFingerprint: null, logSourceGeneration: null }); + + await vi.waitFor(() => { + expect(tracker.disableTracking).toHaveBeenCalledWith('demo', 'stall_monitor'); + }); + await expect(registry.listActiveTeams()).resolves.toEqual([]); + }); + + it('does not activate a team when a reconcile resumes after stop', async () => { + const aliveTeams = createDeferred(); + const tracker = { + enableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), + disableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), + }; + const registry = new ActiveTeamRegistry( + { listAliveProcessTeams: vi.fn(() => aliveTeams.promise) }, + tracker as never + ); + + const reconcilePromise = registry.reconcile(); + await registry.stop(); + aliveTeams.resolve(['demo']); + await reconcilePromise; + + expect(tracker.enableTracking).not.toHaveBeenCalled(); + await expect(registry.listActiveTeams()).resolves.toEqual([]); + }); + it('does not re-enable tracking for teams that are already active during reconcile', async () => { const tracker = { enableTracking: vi.fn(async () => ({ projectFingerprint: null, logSourceGeneration: null })), diff --git a/test/main/services/team/stallMonitor/TeamTaskStallJournal.test.ts b/test/main/services/team/stallMonitor/TeamTaskStallJournal.test.ts index 5e7f306d..d4a4b0b4 100644 --- a/test/main/services/team/stallMonitor/TeamTaskStallJournal.test.ts +++ b/test/main/services/team/stallMonitor/TeamTaskStallJournal.test.ts @@ -49,6 +49,99 @@ describe('TeamTaskStallJournal', () => { expect(secondReady).toEqual([evaluation]); }); + it('allows the same stalled epoch to alert again after the cooldown expires', async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'stall-journal-')); + setClaudeBasePathOverride(tmpDir); + await fs.mkdir(path.join(tmpDir, 'teams', 'demo'), { recursive: true }); + + const journal = new TeamTaskStallJournal({ alertCooldownMs: 10 * 60_000 }); + const evaluation = { + status: 'alert', + taskId: 'task-a', + branch: 'work', + signal: 'turn_ended_after_touch', + epochKey: 'task-a:epoch-1', + reason: 'Potential work stall', + } as const; + + await journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:00:00.000Z', + }); + await expect( + journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:01:00.000Z', + }) + ).resolves.toEqual([evaluation]); + await journal.markAlerted('demo', 'task-a:epoch-1', '2026-04-19T12:01:00.000Z'); + + await expect( + journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:05:00.000Z', + }) + ).resolves.toEqual([]); + await expect( + journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:12:00.000Z', + }) + ).resolves.toEqual([evaluation]); + }); + + it('does not suppress a stalled epoch forever when alertedAt is in the future', async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'stall-journal-')); + setClaudeBasePathOverride(tmpDir); + const teamDir = path.join(tmpDir, 'teams', 'demo'); + await fs.mkdir(teamDir, { recursive: true }); + await fs.writeFile( + path.join(teamDir, 'stall-monitor-journal.json'), + JSON.stringify([ + { + epochKey: 'task-a:epoch-1', + teamName: 'demo', + taskId: 'task-a', + branch: 'work', + signal: 'turn_ended_after_touch', + state: 'alerted', + consecutiveScans: 2, + createdAt: '2026-04-19T12:00:00.000Z', + updatedAt: '2026-04-19T12:01:00.000Z', + alertedAt: '2026-04-19T13:00:00.000Z', + }, + ]), + 'utf8' + ); + + const journal = new TeamTaskStallJournal({ alertCooldownMs: 10 * 60_000 }); + const evaluation = { + status: 'alert', + taskId: 'task-a', + branch: 'work', + signal: 'turn_ended_after_touch', + epochKey: 'task-a:epoch-1', + reason: 'Potential work stall', + } as const; + + await expect( + journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:05:00.000Z', + }) + ).resolves.toEqual([evaluation]); + }); + it('does not prune journal entries outside an explicit task scope', async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'stall-journal-')); setClaudeBasePathOverride(tmpDir); @@ -102,6 +195,64 @@ describe('TeamTaskStallJournal', () => { expect(saved.map((entry) => entry.epochKey)).toEqual(['task-codex:epoch-1']); }); + it('backfills member name on existing stall entries before alerting', async () => { + tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'stall-journal-')); + setClaudeBasePathOverride(tmpDir); + const teamDir = path.join(tmpDir, 'teams', 'demo'); + await fs.mkdir(teamDir, { recursive: true }); + const journalPath = path.join(teamDir, 'stall-monitor-journal.json'); + await fs.writeFile( + journalPath, + JSON.stringify([ + { + epochKey: 'task-a:epoch-1', + teamName: 'demo', + taskId: 'task-a', + branch: 'work', + signal: 'turn_ended_after_touch', + state: 'suspected', + consecutiveScans: 1, + createdAt: '2026-04-19T12:00:00.000Z', + updatedAt: '2026-04-19T12:00:00.000Z', + }, + ]), + 'utf8' + ); + + const journal = new TeamTaskStallJournal(); + const evaluation = { + status: 'alert', + taskId: 'task-a', + memberName: 'bob', + branch: 'work', + signal: 'turn_ended_after_touch', + epochKey: 'task-a:epoch-1', + reason: 'Potential work stall', + } as const; + + await expect( + journal.reconcileScan({ + teamName: 'demo', + evaluations: [evaluation], + activeTaskIds: ['task-a'], + now: '2026-04-19T12:10:00.000Z', + }) + ).resolves.toEqual([evaluation]); + + const saved = JSON.parse(await fs.readFile(journalPath, 'utf8')) as Array<{ + epochKey: string; + memberName?: string; + state: string; + }>; + expect(saved).toEqual([ + expect.objectContaining({ + epochKey: 'task-a:epoch-1', + memberName: 'bob', + state: 'alert_ready', + }), + ]); + }); + it('recovers from an invalid journal file on the next scan', async () => { tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), 'stall-journal-')); setClaudeBasePathOverride(tmpDir); diff --git a/test/main/services/team/stallMonitor/TeamTaskStallMonitor.test.ts b/test/main/services/team/stallMonitor/TeamTaskStallMonitor.test.ts index 82209636..dbfe3f5b 100644 --- a/test/main/services/team/stallMonitor/TeamTaskStallMonitor.test.ts +++ b/test/main/services/team/stallMonitor/TeamTaskStallMonitor.test.ts @@ -2,6 +2,32 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; import { TeamTaskStallMonitor } from '../../../../../src/main/services/team/stallMonitor/TeamTaskStallMonitor'; +function neverResolves(): Promise { + return new Promise(() => undefined); +} + +interface Deferred { + promise: Promise; + resolve: (value: T) => void; + reject: (reason?: unknown) => void; +} + +function createDeferred(): Deferred { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +async function flushAsyncWork(): Promise { + for (let i = 0; i < 8; i += 1) { + await Promise.resolve(); + } +} + describe('TeamTaskStallMonitor', () => { afterEach(() => { vi.useRealTimers(); @@ -113,6 +139,200 @@ describe('TeamTaskStallMonitor', () => { ); }); + it('times out a hung scan so later stall scans continue', async () => { + vi.useFakeTimers(); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_SCAN_INTERVAL_MS', '1000'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_STARTUP_GRACE_MS', '1'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_ACTIVATION_GRACE_MS', '1'); + + const snapshotSource = { + getSnapshot: vi.fn().mockImplementationOnce(neverResolves).mockResolvedValueOnce(null), + }; + const monitor = new TeamTaskStallMonitor( + { + start: vi.fn(), + stop: vi.fn(async () => undefined), + noteTeamChange: vi.fn(), + listActiveTeams: vi.fn(async () => ['demo']), + } as never, + snapshotSource as never, + { evaluateWork: vi.fn(), evaluateReview: vi.fn() } as never, + { reconcileScan: vi.fn(), markAlerted: vi.fn() } as never, + { notifyLead: vi.fn(), notifyOpenCodeOwners: vi.fn() } as never, + { scanTimeoutMs: 10 } + ); + + monitor.start(); + await vi.advanceTimersByTimeAsync(3_010); + expect(snapshotSource.getSnapshot).toHaveBeenCalledTimes(1); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'task stall monitor scan timed out after 10ms' + ); + vi.mocked(console.warn).mockClear(); + + await vi.advanceTimersByTimeAsync(1_001); + expect(snapshotSource.getSnapshot).toHaveBeenCalledTimes(2); + + await monitor.stop(); + }); + + it('does not let one stuck team block stall scans for other active teams', async () => { + vi.useFakeTimers(); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_SCAN_INTERVAL_MS', '1000'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_STARTUP_GRACE_MS', '1'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_ACTIVATION_GRACE_MS', '1'); + + const task = { + id: 'task-healthy', + displayId: 'beef1234', + subject: 'Healthy team task', + }; + const readyEvaluation = { + status: 'alert', + taskId: 'task-healthy', + branch: 'work', + signal: 'turn_ended_after_touch', + epochKey: 'task-healthy:epoch', + reason: 'Potential work stall.', + }; + const snapshotSource = { + getSnapshot: vi.fn(async (teamName: string) => { + if (teamName === 'stuck') { + return neverResolves(); + } + return { + teamName: 'healthy', + inProgressTasks: [task], + reviewOpenTasks: [], + allTasksById: new Map([['task-healthy', task]]), + }; + }), + }; + const journal = { + reconcileScan: vi.fn(async () => [readyEvaluation]), + markAlerted: vi.fn(async () => undefined), + }; + const notifier = { + notifyLead: vi.fn(async () => undefined), + notifyOpenCodeOwners: vi.fn(async () => []), + }; + const monitor = new TeamTaskStallMonitor( + { + start: vi.fn(), + stop: vi.fn(async () => undefined), + noteTeamChange: vi.fn(), + listActiveTeams: vi.fn(async () => ['stuck', 'healthy']), + } as never, + snapshotSource as never, + { + evaluateWork: vi.fn(() => readyEvaluation), + evaluateReview: vi.fn(), + } as never, + journal as never, + notifier as never, + { scanTimeoutMs: 100 } + ); + + monitor.start(); + await vi.advanceTimersByTimeAsync(3_100); + await flushAsyncWork(); + + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'task stall monitor scan timed out after 100ms' + ); + vi.mocked(console.warn).mockClear(); + expect(snapshotSource.getSnapshot).toHaveBeenCalledWith('stuck'); + expect(snapshotSource.getSnapshot).toHaveBeenCalledWith('healthy'); + expect(notifier.notifyLead).toHaveBeenCalledWith( + 'healthy', + expect.arrayContaining([ + expect.objectContaining({ + taskId: 'task-healthy', + }), + ]) + ); + expect(journal.markAlerted).toHaveBeenCalledWith( + 'healthy', + 'task-healthy:epoch', + expect.any(String) + ); + + await monitor.stop(); + }); + + it('ignores late side effects from a scan that already timed out', async () => { + vi.useFakeTimers(); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_SCAN_INTERVAL_MS', '1000'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_STARTUP_GRACE_MS', '1'); + vi.stubEnv('CLAUDE_TEAM_TASK_STALL_ACTIVATION_GRACE_MS', '1'); + + const staleJournalScan = createDeferred(); + const readyEvaluation = { + status: 'alert', + taskId: 'task-a', + branch: 'work', + signal: 'turn_ended_after_touch', + epochKey: 'task-a:epoch', + reason: 'Potential work stall.', + }; + const task = { id: 'task-a', displayId: 'abcd1234', subject: 'Task A' }; + const notifier = { + notifyLead: vi.fn(async () => undefined), + notifyOpenCodeOwners: vi.fn(async () => []), + }; + const journal = { + reconcileScan: vi + .fn() + .mockImplementationOnce(() => staleJournalScan.promise) + .mockResolvedValueOnce([]), + markAlerted: vi.fn(async () => undefined), + }; + const monitor = new TeamTaskStallMonitor( + { + start: vi.fn(), + stop: vi.fn(async () => undefined), + noteTeamChange: vi.fn(), + listActiveTeams: vi.fn(async () => ['demo']), + } as never, + { + getSnapshot: vi.fn(async () => ({ + teamName: 'demo', + inProgressTasks: [task], + reviewOpenTasks: [], + allTasksById: new Map([['task-a', task]]), + })), + } as never, + { + evaluateWork: vi.fn(() => readyEvaluation), + evaluateReview: vi.fn(), + } as never, + journal as never, + notifier as never, + { scanTimeoutMs: 10 } + ); + + monitor.start(); + await vi.advanceTimersByTimeAsync(3_010); + expect(journal.reconcileScan).toHaveBeenCalledTimes(1); + + await vi.advanceTimersByTimeAsync(10); + expect(vi.mocked(console.warn).mock.calls[0]?.join(' ')).toContain( + 'task stall monitor scan timed out after 10ms' + ); + vi.mocked(console.warn).mockClear(); + + await vi.advanceTimersByTimeAsync(1_001); + expect(journal.reconcileScan).toHaveBeenCalledTimes(2); + + staleJournalScan.resolve([readyEvaluation]); + await flushAsyncWork(); + + expect(notifier.notifyLead).not.toHaveBeenCalled(); + expect(journal.markAlerted).not.toHaveBeenCalled(); + + await monitor.stop(); + }); + it('defaults to OpenCode owner remediation without duplicate lead alerts when remediation is accepted', async () => { vi.useFakeTimers(); vi.stubEnv('CLAUDE_TEAM_TASK_STALL_SCAN_INTERVAL_MS', '1000'); diff --git a/test/renderer/components/runtime/ProviderRuntimeSettingsDialog.test.ts b/test/renderer/components/runtime/ProviderRuntimeSettingsDialog.test.ts index c9471c76..12e2d37a 100644 --- a/test/renderer/components/runtime/ProviderRuntimeSettingsDialog.test.ts +++ b/test/renderer/components/runtime/ProviderRuntimeSettingsDialog.test.ts @@ -19,6 +19,11 @@ interface StoreState { }; codex: { preferredAuthMode: 'auto' | 'chatgpt' | 'api_key'; + customProvider: { + enabled: boolean; + baseUrl: string; + model: string; + }; }; }; }; @@ -115,6 +120,25 @@ vi.mock('@renderer/components/ui/button', () => ({ ), })); +vi.mock('@renderer/components/ui/checkbox', () => ({ + Checkbox: ({ + checked, + disabled, + onCheckedChange, + }: { + checked?: boolean; + disabled?: boolean; + onCheckedChange?: (checked: boolean) => void; + }) => + React.createElement('input', { + type: 'checkbox', + checked: Boolean(checked), + disabled, + onChange: (event: React.ChangeEvent) => + onCheckedChange?.(event.currentTarget.checked), + }), +})); + vi.mock('@renderer/components/ui/dialog', () => ({ Dialog: ({ open, children }: React.PropsWithChildren<{ open: boolean }>) => open ? React.createElement('div', { 'data-testid': 'dialog' }, children) : null, @@ -282,6 +306,13 @@ function createCodexProvider( Boolean(overrides?.authenticated ?? true) || Boolean(overrides?.apiKeyConfigured) ? 'ready_api_key' : 'missing_auth', + customProvider: { + enabled: false, + active: false, + baseUrl: '', + model: '', + issueMessage: null, + }, ...overrides?.codex, }, }, @@ -487,6 +518,11 @@ describe('ProviderRuntimeSettingsDialog', () => { }, codex: { preferredAuthMode: 'auto', + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, }, }, }; @@ -518,6 +554,10 @@ describe('ProviderRuntimeSettingsDialog', () => { codex: { ...storeState.appConfig.providerConnections.codex, ...(nextProviderConnections.codex ?? {}), + customProvider: { + ...storeState.appConfig.providerConnections.codex.customProvider, + ...(nextProviderConnections.codex?.customProvider ?? {}), + }, }, }, }; @@ -997,6 +1037,166 @@ describe('ProviderRuntimeSettingsDialog', () => { expect(host.textContent).toContain('Connect ChatGPT'); }); + it('saves a Codex custom provider profile and reuses OPENAI_API_KEY storage', async () => { + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const onRefreshProvider = vi.fn(() => Promise.resolve(undefined)); + + await act(async () => { + root.render( + React.createElement(ProviderRuntimeSettingsDialog, { + open: true, + onOpenChange: vi.fn(), + providers: [ + createCodexProvider({ + authenticated: false, + authMethod: null, + apiKeyConfigured: false, + apiKeySource: null, + apiKeySourceLabel: null, + }), + ], + initialProviderId: 'codex', + onSelectBackend: vi.fn(), + onRefreshProvider, + }) + ); + await Promise.resolve(); + }); + + expect(host.textContent).toContain('Custom API endpoint'); + const enabledInput = host.querySelector( + '[data-testid="codex-custom-provider-panel"] input[type="checkbox"]' + ) as HTMLInputElement | null; + const baseUrlInput = host.querySelector( + '[data-testid="codex-custom-provider-base-url"]' + ) as HTMLInputElement | null; + const modelInput = host.querySelector( + '[data-testid="codex-custom-provider-model"]' + ) as HTMLInputElement | null; + const apiKeyInput = host.querySelector( + '[data-testid="codex-custom-provider-api-key"]' + ) as HTMLInputElement | null; + expect(enabledInput).not.toBeNull(); + expect(baseUrlInput).not.toBeNull(); + expect(modelInput).not.toBeNull(); + expect(apiKeyInput).not.toBeNull(); + + await act(async () => { + enabledInput!.click(); + setInputValue(baseUrlInput!, 'https://gateway.example.com/v1'); + setInputValue(modelInput!, 'gateway-codex-model'); + setInputValue(apiKeyInput!, 'sk-test'); + await Promise.resolve(); + }); + + await act(async () => { + findButtonByText(host, 'Save endpoint').click(); + await Promise.resolve(); + }); + + expect(storeState.saveApiKey).toHaveBeenCalledWith({ + id: undefined, + name: 'Codex API Key', + envVarName: 'OPENAI_API_KEY', + value: 'sk-test', + scope: 'user', + }); + expect(storeState.updateConfig).toHaveBeenCalledWith('providerConnections', { + codex: { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }, + }); + expect(codexAccountHookState.refresh).toHaveBeenCalledWith({ + includeRateLimits: true, + forceRefreshToken: true, + }); + expect(onRefreshProvider).toHaveBeenCalledWith('codex'); + }); + + it('disables Codex custom provider without deleting its saved key or profile fields', async () => { + const host = document.createElement('div'); + document.body.appendChild(host); + const root = createRoot(host); + const onRefreshProvider = vi.fn(() => Promise.resolve(undefined)); + storeState.appConfig.providerConnections.codex = { + preferredAuthMode: 'api_key', + customProvider: { + enabled: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }; + storeState.apiKeys = [ + { + id: 'openai-key', + envVarName: 'OPENAI_API_KEY', + scope: 'user', + name: 'Codex API Key', + maskedValue: 'sk-...xyz', + }, + ]; + + await act(async () => { + root.render( + React.createElement(ProviderRuntimeSettingsDialog, { + open: true, + onOpenChange: vi.fn(), + providers: [ + createCodexProvider({ + authenticated: true, + authMethod: 'api_key', + configuredAuthMode: 'api_key', + apiKeyConfigured: true, + apiKeySource: 'stored', + apiKeySourceLabel: 'Stored in app', + codex: { + preferredAuthMode: 'api_key', + effectiveAuthMode: 'api_key', + customProvider: { + enabled: true, + active: true, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + issueMessage: null, + }, + }, + }), + ], + initialProviderId: 'codex', + onSelectBackend: vi.fn(), + onRefreshProvider, + }) + ); + await Promise.resolve(); + }); + + expect(host.textContent).toContain('sk-...xyz'); + + await act(async () => { + findButtonByText(host, 'Disable').click(); + await Promise.resolve(); + }); + + expect(storeState.updateConfig).toHaveBeenCalledWith('providerConnections', { + codex: { + customProvider: { + enabled: false, + baseUrl: 'https://gateway.example.com/v1', + model: 'gateway-codex-model', + }, + }, + }); + expect(storeState.deleteApiKey).not.toHaveBeenCalled(); + expect(onRefreshProvider).toHaveBeenCalledWith('codex'); + }); + it('explains the missing Codex ChatGPT login without mixing it up with the detected API key', async () => { const host = document.createElement('div'); document.body.appendChild(host); diff --git a/test/renderer/store/extensionsSlice.test.ts b/test/renderer/store/extensionsSlice.test.ts index ce787ee6..1b0ce402 100644 --- a/test/renderer/store/extensionsSlice.test.ts +++ b/test/renderer/store/extensionsSlice.test.ts @@ -255,6 +255,11 @@ function makeAppConfig(multimodelEnabled: boolean): AppConfig { }, codex: { preferredAuthMode: 'auto', + customProvider: { + enabled: false, + baseUrl: '', + model: '', + }, }, }, runtime: {