From f81ed03b751f4e4ea6c010e7b011d2f656219d9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 6 Jan 2026 18:28:32 +0100 Subject: [PATCH 01/13] E-RC: added commented fix to camera client eval dict, may be needed if image is too large --- src/instamatic/camera/camera_client.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/instamatic/camera/camera_client.py b/src/instamatic/camera/camera_client.py index 9f75314d..79af5d43 100644 --- a/src/instamatic/camera/camera_client.py +++ b/src/instamatic/camera/camera_client.py @@ -38,6 +38,23 @@ def start_server_in_subprocess(): atexit.register(kill_server, p) +# +# def recv_pickle(sock: socket.socket, timeout: float = 60.0): +# import pickle +# +# buffer = b'' +# t0 = time.perf_counter() +# while time.perf_counter() - t0 < timeout: +# buffer += sock.recv(BUFSIZE) +# if not buffer: +# raise EOFError('Socket closed before pickle was complete') +# try: +# return pickle.loads(buffer) +# except (EOFError, pickle.UnpicklingError): +# continue +# return (500, None) # after timeout + + class CamClient: """Simulates a Camera object and synchronizes calls over a socket server. @@ -134,6 +151,7 @@ def _eval_dct(self, dct): if acquiring_image and not self.use_shared_memory: response = self.s.recv(self._imagebufsize) + # response = recv_pickle(self.s) # may be needed if image is too large else: response = self.s.recv(self._bufsize) From 55f67b4f0c2dcc4c5782346a271bafb58beece92 Mon Sep 17 00:00:00 2001 From: Daniel Tchon Date: Tue, 6 Jan 2026 18:34:21 +0100 Subject: [PATCH 02/13] Attempts at hot-fixing beamshift calibration --- src/instamatic/calibrate/calibrate_beamshift.py | 4 ++++ src/instamatic/gui/ctrl_frame.py | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/instamatic/calibrate/calibrate_beamshift.py b/src/instamatic/calibrate/calibrate_beamshift.py index de2f9066..e99fd865 100644 --- a/src/instamatic/calibrate/calibrate_beamshift.py +++ b/src/instamatic/calibrate/calibrate_beamshift.py @@ -116,6 +116,8 @@ def to_file(self, fn: AnyPath = CALIB_BEAMSHIFT, outdir: AnyPath = '.') -> None: def plot(self, to_file: Optional[AnyPath] = None): """Assuming the data is present, plot the data.""" shifts = np.dot(self.shifts, np.linalg.inv(self.transform)) + print(self.pixels.T) + print(shifts.T) plt.scatter(*self.pixels.T, marker='>', label='Observed pixel shifts') plt.scatter(*shifts.T, marker='<', label='Reconstructed pixel shifts') plt.legend() @@ -136,6 +138,8 @@ def annotate_videostream(self, vsp: Optional[VideoStreamProcessor] = None) -> No for p, s in zip(self.pixels, shifts): p = (p + self.reference_pixel)[::-1] # xy coords inverted for plot s = (s + self.reference_pixel)[::-1] # xy coords inverted for plot + p /= 4 # HARD-CODED binning because I don't have time to do it nicely + s /= 4 # HARD-CODED binning because I don't have time to do it nicely ins.append(vsp.draw.circle(p, radius=3, fill='blue')) ins.append(vsp.draw.circle(s, radius=3, fill='orange')) ins.append(vsp.draw.circle(self.reference_pixel[::-1], radius=3, fill='black')) diff --git a/src/instamatic/gui/ctrl_frame.py b/src/instamatic/gui/ctrl_frame.py index ee4a4360..ca51e6e2 100644 --- a/src/instamatic/gui/ctrl_frame.py +++ b/src/instamatic/gui/ctrl_frame.py @@ -363,10 +363,14 @@ def toggle_rmb_beam(self, _name, _index, _mode) -> None: print('Run `instamatic.calibrate_beamshift` there to use this feature.') self.var_rmb_beam.set(False) return + + binning = self.ctrl.cam.default_binsize def _callback(click: ClickEvent) -> None: if click.button == MouseButton.RIGHT: - bs = calib_beamshift.pixelcoord_to_beamshift((click.y, click.x)) + pixel_x = click.x * binning + pixel_y = click.y * binning + bs = calib_beamshift.pixelcoord_to_beamshift((pixel_x, pixel_y)) self.ctrl.beamshift.set(*[float(b) for b in bs]) d.add_listener('rmb_beam', _callback, active=True) From 24cc9fa9644504d8248a3f4328a9283c5bced78a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 7 Jan 2026 09:36:05 +0100 Subject: [PATCH 03/13] E-RC: take into account the factor of binning when tracking @ FastADT --- src/instamatic/experiments/fast_adt/experiment.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/instamatic/experiments/fast_adt/experiment.py b/src/instamatic/experiments/fast_adt/experiment.py index ffc57750..141c9c9b 100644 --- a/src/instamatic/experiments/fast_adt/experiment.py +++ b/src/instamatic/experiments/fast_adt/experiment.py @@ -213,6 +213,7 @@ def __init__( self.flatfield = flatfield self.fast_adt_frame = experiment_frame self.beamshift: Optional[CalibBeamShift] = None + self.binsize: int = 1 self.camera_length: int = 0 if videostream_frame is not None: @@ -337,6 +338,7 @@ def start_collection(self, **params) -> None: with self.ctrl.beam.blanked(), self.ctrl.cam.blocked(): if params['tracking_algo'] == 'manual': + self.binsize = self.ctrl.cam.default_binsize self.runs.tracking = TrackingRun.from_params(params) self.determine_pathing_manually() for pathing_run in self.runs.pathing: @@ -388,7 +390,7 @@ def determine_pathing_manually(self) -> None: self.beamshift = self.get_beamshift() self.msg1('Locate the beam (move it if needed) and click on its center.') with self.click_listener as cl: - obs_beampixel_xy = np.array(cl.get_click().xy) + obs_beampixel_xy = np.array(cl.get_click().xy) * self.binsize cal_beampixel_yx = self.beamshift.beamshift_to_pixelcoord(self.ctrl.beamshift.get()) self.ctrl.restore('FastADT_track') @@ -401,11 +403,12 @@ def determine_pathing_manually(self) -> None: self.msg1(f'Click on tracked point: {step.summary}.') with self.displayed_pathing(step=step), self.click_listener: click = self.click_listener.get_click() - delta_yx = (np.array(click.xy) - obs_beampixel_xy)[::-1] + click_xy = np.array(click.xy) * self.binsize + delta_yx = (click_xy - obs_beampixel_xy)[::-1] click_beampixel_yx = cast(Sequence[float], cal_beampixel_yx + delta_yx) click_beamshift_xy = self.beamshift.pixelcoord_to_beamshift(click_beampixel_yx) cols = ['beampixel_x', 'beampixel_y', 'beamshift_x', 'beamshift_y'] - run.table.loc[step.Index, cols] = *click.xy, *click_beamshift_xy + run.table.loc[step.Index, cols] = *click_xy, *click_beamshift_xy tracking_frames.append(step.image) if 'image' not in run.table: run.table['image'] = tracking_frames From 7d9496b53add40796c5e0bf4792399bbb772adf2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 7 Jan 2026 09:49:07 +0100 Subject: [PATCH 04/13] Revert "E-RC: take into account the factor of binning when tracking @ FastADT" This reverts commit 24cc9fa9644504d8248a3f4328a9283c5bced78a. --- src/instamatic/experiments/fast_adt/experiment.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/instamatic/experiments/fast_adt/experiment.py b/src/instamatic/experiments/fast_adt/experiment.py index 141c9c9b..ffc57750 100644 --- a/src/instamatic/experiments/fast_adt/experiment.py +++ b/src/instamatic/experiments/fast_adt/experiment.py @@ -213,7 +213,6 @@ def __init__( self.flatfield = flatfield self.fast_adt_frame = experiment_frame self.beamshift: Optional[CalibBeamShift] = None - self.binsize: int = 1 self.camera_length: int = 0 if videostream_frame is not None: @@ -338,7 +337,6 @@ def start_collection(self, **params) -> None: with self.ctrl.beam.blanked(), self.ctrl.cam.blocked(): if params['tracking_algo'] == 'manual': - self.binsize = self.ctrl.cam.default_binsize self.runs.tracking = TrackingRun.from_params(params) self.determine_pathing_manually() for pathing_run in self.runs.pathing: @@ -390,7 +388,7 @@ def determine_pathing_manually(self) -> None: self.beamshift = self.get_beamshift() self.msg1('Locate the beam (move it if needed) and click on its center.') with self.click_listener as cl: - obs_beampixel_xy = np.array(cl.get_click().xy) * self.binsize + obs_beampixel_xy = np.array(cl.get_click().xy) cal_beampixel_yx = self.beamshift.beamshift_to_pixelcoord(self.ctrl.beamshift.get()) self.ctrl.restore('FastADT_track') @@ -403,12 +401,11 @@ def determine_pathing_manually(self) -> None: self.msg1(f'Click on tracked point: {step.summary}.') with self.displayed_pathing(step=step), self.click_listener: click = self.click_listener.get_click() - click_xy = np.array(click.xy) * self.binsize - delta_yx = (click_xy - obs_beampixel_xy)[::-1] + delta_yx = (np.array(click.xy) - obs_beampixel_xy)[::-1] click_beampixel_yx = cast(Sequence[float], cal_beampixel_yx + delta_yx) click_beamshift_xy = self.beamshift.pixelcoord_to_beamshift(click_beampixel_yx) cols = ['beampixel_x', 'beampixel_y', 'beamshift_x', 'beamshift_y'] - run.table.loc[step.Index, cols] = *click_xy, *click_beamshift_xy + run.table.loc[step.Index, cols] = *click.xy, *click_beamshift_xy tracking_frames.append(step.image) if 'image' not in run.table: run.table['image'] = tracking_frames From eee353b5d2d9266c193e3add779c17af9ef8e4d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 7 Jan 2026 09:58:15 +0100 Subject: [PATCH 05/13] E-RC: take into account the factor of binning when tracking @ FastADT + display them properly --- src/instamatic/experiments/fast_adt/experiment.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/instamatic/experiments/fast_adt/experiment.py b/src/instamatic/experiments/fast_adt/experiment.py index ffc57750..616484aa 100644 --- a/src/instamatic/experiments/fast_adt/experiment.py +++ b/src/instamatic/experiments/fast_adt/experiment.py @@ -213,6 +213,7 @@ def __init__( self.flatfield = flatfield self.fast_adt_frame = experiment_frame self.beamshift: Optional[CalibBeamShift] = None + self.binsize: int = 1 self.camera_length: int = 0 if videostream_frame is not None: @@ -337,6 +338,7 @@ def start_collection(self, **params) -> None: with self.ctrl.beam.blanked(), self.ctrl.cam.blocked(): if params['tracking_algo'] == 'manual': + self.binsize = self.ctrl.cam.default_binsize self.runs.tracking = TrackingRun.from_params(params) self.determine_pathing_manually() for pathing_run in self.runs.pathing: @@ -366,8 +368,8 @@ def displayed_pathing(self, step: Step) -> None: draw = self.videostream_processor.draw instructions: list[draw.Instruction] = [] for run_i, p in enumerate(self.runs.pathing): - x = p.table.at[step.Index, 'beampixel_x'] - y = p.table.at[step.Index, 'beampixel_y'] + x = p.table.at[step.Index, 'beampixel_x'] / self.binsize + y = p.table.at[step.Index, 'beampixel_y'] / self.binsize instructions.append(draw.circle((x, y), fill='white', radius=5)) instructions.append(draw.circle((x, y), fill=get_color(run_i), radius=3)) try: @@ -388,7 +390,7 @@ def determine_pathing_manually(self) -> None: self.beamshift = self.get_beamshift() self.msg1('Locate the beam (move it if needed) and click on its center.') with self.click_listener as cl: - obs_beampixel_xy = np.array(cl.get_click().xy) + obs_beampixel_xy = np.array(cl.get_click().xy) * self.binsize cal_beampixel_yx = self.beamshift.beamshift_to_pixelcoord(self.ctrl.beamshift.get()) self.ctrl.restore('FastADT_track') @@ -401,11 +403,12 @@ def determine_pathing_manually(self) -> None: self.msg1(f'Click on tracked point: {step.summary}.') with self.displayed_pathing(step=step), self.click_listener: click = self.click_listener.get_click() - delta_yx = (np.array(click.xy) - obs_beampixel_xy)[::-1] + click_xy = np.array(click.xy) * self.binsize + delta_yx = (click_xy - obs_beampixel_xy)[::-1] click_beampixel_yx = cast(Sequence[float], cal_beampixel_yx + delta_yx) click_beamshift_xy = self.beamshift.pixelcoord_to_beamshift(click_beampixel_yx) cols = ['beampixel_x', 'beampixel_y', 'beamshift_x', 'beamshift_y'] - run.table.loc[step.Index, cols] = *click.xy, *click_beamshift_xy + run.table.loc[step.Index, cols] = *click_xy, *click_beamshift_xy tracking_frames.append(step.image) if 'image' not in run.table: run.table['image'] = tracking_frames From fc8a2172a393be1716d0f7d5f4007c37981b16f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 7 Jan 2026 10:27:18 +0100 Subject: [PATCH 06/13] E-RC: fix the coordinate order in ctrl frame toggle_rmb_beam callback --- src/instamatic/gui/ctrl_frame.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/instamatic/gui/ctrl_frame.py b/src/instamatic/gui/ctrl_frame.py index ca51e6e2..631ec500 100644 --- a/src/instamatic/gui/ctrl_frame.py +++ b/src/instamatic/gui/ctrl_frame.py @@ -363,14 +363,14 @@ def toggle_rmb_beam(self, _name, _index, _mode) -> None: print('Run `instamatic.calibrate_beamshift` there to use this feature.') self.var_rmb_beam.set(False) return - + binning = self.ctrl.cam.default_binsize def _callback(click: ClickEvent) -> None: if click.button == MouseButton.RIGHT: pixel_x = click.x * binning pixel_y = click.y * binning - bs = calib_beamshift.pixelcoord_to_beamshift((pixel_x, pixel_y)) + bs = calib_beamshift.pixelcoord_to_beamshift((pixel_y, pixel_x)) self.ctrl.beamshift.set(*[float(b) for b in bs]) d.add_listener('rmb_beam', _callback, active=True) From d5c82947a4703b0877ff18057a35ba594263d5e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Wed, 7 Jan 2026 12:47:25 +0100 Subject: [PATCH 07/13] E-RC: Refactor annotate_videostream to use binsize parameter --- src/instamatic/calibrate/calibrate_beamshift.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/instamatic/calibrate/calibrate_beamshift.py b/src/instamatic/calibrate/calibrate_beamshift.py index e99fd865..8771a9f4 100644 --- a/src/instamatic/calibrate/calibrate_beamshift.py +++ b/src/instamatic/calibrate/calibrate_beamshift.py @@ -101,7 +101,8 @@ def live( ) -> Self: while True: c = calibrate_beamshift(ctrl=ctrl, save_images=True, outdir=outdir) - with c.annotate_videostream(vsp) if vsp else nullcontext(): + binsize = ctrl.cam.default_binsize + with c.annotate_videostream(vsp, binsize) if vsp else nullcontext(): if input(' >> Accept? [y/n] ') == 'y': return c @@ -129,17 +130,15 @@ def plot(self, to_file: Optional[AnyPath] = None): plt.show() @contextmanager - def annotate_videostream(self, vsp: Optional[VideoStreamProcessor] = None) -> None: + def annotate_videostream(self, vsp: VideoStreamProcessor, binsize: int = 1) -> None: shifts = np.dot(self.shifts, np.linalg.inv(self.transform)) ins: list[DeferredImageDraw.Instruction] = [] vsp.temporary_frame = np.max(self.images, axis=0) print('Determined (blue) vs calibrated (orange) beam positions:') for p, s in zip(self.pixels, shifts): - p = (p + self.reference_pixel)[::-1] # xy coords inverted for plot - s = (s + self.reference_pixel)[::-1] # xy coords inverted for plot - p /= 4 # HARD-CODED binning because I don't have time to do it nicely - s /= 4 # HARD-CODED binning because I don't have time to do it nicely + p = (p + self.reference_pixel)[::-1] / binsize # xy coords inverted for plot + s = (s + self.reference_pixel)[::-1] / binsize # xy coords inverted for plot ins.append(vsp.draw.circle(p, radius=3, fill='blue')) ins.append(vsp.draw.circle(s, radius=3, fill='orange')) ins.append(vsp.draw.circle(self.reference_pixel[::-1], radius=3, fill='black')) From 208e1141af6ea8f543d96f5d689bf850bce53ee7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Thu, 8 Jan 2026 11:59:55 +0100 Subject: [PATCH 08/13] E-RC: patch instamatic RPC to allow get_movie generator --- src/instamatic/camera/camera_client.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/instamatic/camera/camera_client.py b/src/instamatic/camera/camera_client.py index 79af5d43..895cac18 100644 --- a/src/instamatic/camera/camera_client.py +++ b/src/instamatic/camera/camera_client.py @@ -6,6 +6,7 @@ import threading import time from functools import wraps +from typing import Any, Generator import numpy as np @@ -143,11 +144,7 @@ def _eval_dct(self, dct): with self._eval_lock: self.s.send(dumper(dct)) - acquiring_image = dct['attr_name'] == 'get_image' - acquiring_movie = dct['attr_name'] == 'get_movie' - - if acquiring_movie: - raise NotImplementedError('Acquiring movies over a socket is not supported.') + acquiring_image = dct['attr_name'] in {'get_image', 'get_movie'} if acquiring_image and not self.use_shared_memory: response = self.s.recv(self._imagebufsize) @@ -164,6 +161,8 @@ def _eval_dct(self, dct): data = self.get_data_from_shared_memory(**data) if status == 200: + if isinstance(data, dict) and '__generator__' in data: + return self._wrap_remote_generator(data['__generator__']) return data elif status == 500: @@ -224,3 +223,20 @@ def block(self): def unblock(self): raise NotImplementedError('This camera cannot be streamed.') + + def _wrap_remote_generator(self, gen_id: str) -> Generator[Any]: + """Pass a reference to yield from a remote __generator__ with id.""" + + def generator(): + kwargs = {'id': gen_id} + try: + while True: + dct = {'attr_name': '__gen_next__', 'kwargs': kwargs} + status, value = self._eval_dct(dct) + if status == 204: + return + yield value + finally: + self._eval_dct({'attr_name': '__gen_close__', 'kwargs': kwargs}) + + return generator() From 7a159c4922c9da949c9953e7639dd016e9e51c96 Mon Sep 17 00:00:00 2001 From: Daniel Tchon Date: Fri, 9 Jan 2026 08:57:57 +0100 Subject: [PATCH 09/13] E-RC: Fixes to type passed, generator unpacking --- src/instamatic/camera/camera_client.py | 6 +++--- src/instamatic/camera/videostream.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/instamatic/camera/camera_client.py b/src/instamatic/camera/camera_client.py index 895cac18..08fdf463 100644 --- a/src/instamatic/camera/camera_client.py +++ b/src/instamatic/camera/camera_client.py @@ -144,7 +144,7 @@ def _eval_dct(self, dct): with self._eval_lock: self.s.send(dumper(dct)) - acquiring_image = dct['attr_name'] in {'get_image', 'get_movie'} + acquiring_image = dct['attr_name'] in {'get_image', 'get_movie', '__gen_next__'} if acquiring_image and not self.use_shared_memory: response = self.s.recv(self._imagebufsize) @@ -232,8 +232,8 @@ def generator(): try: while True: dct = {'attr_name': '__gen_next__', 'kwargs': kwargs} - status, value = self._eval_dct(dct) - if status == 204: + value = self._eval_dct(dct) + if value is None: return yield value finally: diff --git a/src/instamatic/camera/videostream.py b/src/instamatic/camera/videostream.py index 858cde89..43c97b6e 100644 --- a/src/instamatic/camera/videostream.py +++ b/src/instamatic/camera/videostream.py @@ -72,8 +72,8 @@ def run(self): if self.acquireInitiateEvent.is_set(): r = self.request self.acquireInitiateEvent.clear() - e = r.exposure if r.exposure else self.default_exposure - b = r.binsize if r.binsize else self.default_binsize + e = float(r.exposure if r.exposure else self.default_exposure) + b = int(r.binsize if r.binsize else self.default_binsize) if isinstance(r, ImageRequest): media = self.cam.get_image(exposure=e, binsize=b) self.callback(media, request=r) From 38d35f66392bf5d3d32a399c2ca18642db1addc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 12 Jan 2026 20:19:54 +0100 Subject: [PATCH 10/13] ER-C: remove unused socket protocol, comments, debug statements --- .../calibrate/calibrate_beamshift.py | 2 -- src/instamatic/camera/camera_client.py | 18 ------------------ 2 files changed, 20 deletions(-) diff --git a/src/instamatic/calibrate/calibrate_beamshift.py b/src/instamatic/calibrate/calibrate_beamshift.py index 8771a9f4..f440aa44 100644 --- a/src/instamatic/calibrate/calibrate_beamshift.py +++ b/src/instamatic/calibrate/calibrate_beamshift.py @@ -117,8 +117,6 @@ def to_file(self, fn: AnyPath = CALIB_BEAMSHIFT, outdir: AnyPath = '.') -> None: def plot(self, to_file: Optional[AnyPath] = None): """Assuming the data is present, plot the data.""" shifts = np.dot(self.shifts, np.linalg.inv(self.transform)) - print(self.pixels.T) - print(shifts.T) plt.scatter(*self.pixels.T, marker='>', label='Observed pixel shifts') plt.scatter(*shifts.T, marker='<', label='Reconstructed pixel shifts') plt.legend() diff --git a/src/instamatic/camera/camera_client.py b/src/instamatic/camera/camera_client.py index 08fdf463..a0591d05 100644 --- a/src/instamatic/camera/camera_client.py +++ b/src/instamatic/camera/camera_client.py @@ -39,23 +39,6 @@ def start_server_in_subprocess(): atexit.register(kill_server, p) -# -# def recv_pickle(sock: socket.socket, timeout: float = 60.0): -# import pickle -# -# buffer = b'' -# t0 = time.perf_counter() -# while time.perf_counter() - t0 < timeout: -# buffer += sock.recv(BUFSIZE) -# if not buffer: -# raise EOFError('Socket closed before pickle was complete') -# try: -# return pickle.loads(buffer) -# except (EOFError, pickle.UnpicklingError): -# continue -# return (500, None) # after timeout - - class CamClient: """Simulates a Camera object and synchronizes calls over a socket server. @@ -148,7 +131,6 @@ def _eval_dct(self, dct): if acquiring_image and not self.use_shared_memory: response = self.s.recv(self._imagebufsize) - # response = recv_pickle(self.s) # may be needed if image is too large else: response = self.s.recv(self._bufsize) From a129ce5f9b4e5786b57b6f1682d8a96fb06d3edb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Mon, 12 Jan 2026 20:24:58 +0100 Subject: [PATCH 11/13] ER-C: allow handling generators (get_movie) by the cam server --- src/instamatic/server/cam_server.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/instamatic/server/cam_server.py b/src/instamatic/server/cam_server.py index 290fbf70..6fc299b1 100644 --- a/src/instamatic/server/cam_server.py +++ b/src/instamatic/server/cam_server.py @@ -1,11 +1,13 @@ from __future__ import annotations import datetime +import inspect import logging import queue import socket import threading import traceback +import uuid import numpy as np @@ -19,6 +21,7 @@ if config.settings.cam_use_shared_memory: from multiprocessing import shared_memory +_generators = {} condition = threading.Condition() box = [] @@ -97,6 +100,10 @@ def run(self): try: ret = self.evaluate(attr_name, args, kwargs) status = 200 + if inspect.isgenerator(ret): + gen_id = uuid.uuid4().hex + _generators[gen_id] = ret + ret = {'__generator__': gen_id} except Exception as e: traceback.print_exc() if self.log: @@ -121,7 +128,19 @@ def run(self): def evaluate(self, attr_name: str, args: list, kwargs: dict): """Evaluate the function or attribute `attr_name` on `self.cam`, if `attr_name` refers to a function, call it with *args and **kwargs.""" - # print(attr_name, args, kwargs) + + if attr_name == '__gen_next__': + gen = _generators[kwargs['id']] + try: + return next(gen) + except StopIteration: + del _generators[kwargs['id']] + return + + if attr_name == '__gen_close__': + _generators.pop(kwargs['id'], None) + return + f = getattr(self.cam, attr_name) return f(*args, **kwargs) if callable(f) else f From 5f3fc0905c7d277a2fdfc87b8df635adf66c7deb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 13 Jan 2026 19:37:55 +0100 Subject: [PATCH 12/13] Update documentation with relevant changes from https://sites.google.com/view/instamatic-on-titan --- docs/config.md | 2 +- docs/network.md | 4 +++- docs/setup.md | 10 ++++++---- readme.md | 5 +++-- 4 files changed, 13 insertions(+), 8 deletions(-) diff --git a/docs/config.md b/docs/config.md index 2996b5dd..36760a31 100644 --- a/docs/config.md +++ b/docs/config.md @@ -207,7 +207,7 @@ This file holds the specifications of the camera. This file is must be located t : Give the interface of the camera interface to connect to, for example: `timepix`/`emmenu`/`simulate`/`gatan`/`merlin`. Leave blank or set to `None` to load the camera specs, but do not load the camera module (this also turns off the videostream gui). **dead_time** -: Set the dead time (i.e. the gap between acquisitions) of the detector; if this value (`camera.dead_time`) is not set but required, Instamatic might attempt to use `CalibMovieDelays.dead_time` value calibrated via `instamatic.calibrate_movie_delays` instead. Typically, Instamatic will not run this calibration automatically: the user needs to either set `camera.dead_time` or call `instamatic.calibrate_movie_delays` themselves. +: Set the dead time (i.e. the gap between acquisitions) of the detector. This value is especially important for cameras what work remotely or otherwise feature dead time significant when compared to typical data collection time. If `camera.dead_time` is not set but required, Instamatic might attempt to use `CalibMovieDelays.dead_time` value calibrated via `instamatic.calibrate_movie_delays` instead. Typically, Instamatic will not run this calibration automatically: the user needs to either set `camera.dead_time` or call `instamatic.calibrate_movie_delays` themselves. **default_binsize** : Set the default binsize, default: `1`. diff --git a/docs/network.md b/docs/network.md index 29359889..4888b0c6 100644 --- a/docs/network.md +++ b/docs/network.md @@ -40,7 +40,7 @@ use_cam_server: False ## Example 2 -This is an example where the microscope and camera PCs should be controlled through an intermediate support PC. +In this example, the microscope and camera are controlled by a dedicated computer(s), distinct from an intermediate support PC running the Instamatic GUI. This scenario is preferred if, for any reason, Instamatic can not be fully installed on the Microscope/Camera PC such as when using [instamatic-tecnai-server](https://github.com/instamatic-dev/instamatic-tecnai-server) in lieu of the full Instamatic. If your camera can be controlled directly through TCP/IP, such as the MerlinEM or ASI Cheetah (via `serval`), do not use `instamatic.camserver`, but connect directly to the IP. For example, for Merlin. @@ -72,6 +72,8 @@ cam_server_port: 8088 cam_use_shared_memory: False ``` +A case of a setup where the Microscope PC supports both the TEM and a camera server via the Tecnai server while the main GUI runs on a separate Support PC is partially discussed [here](https://sites.google.com/view/instamatic-on-titan). + ## Example 3 If your camera cannot be controlled through TCP/IP, you might try this solution. This seems to be a common setup for TFS/FEI microscopes. diff --git a/docs/setup.md b/docs/setup.md index 906f155e..84de012b 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -8,7 +8,9 @@ If you are using a JEOL TEM, make sure `instamatic` is installed on a computer w ## FEI -For FEI microscopes, `instamatic` must be installed on the microscope control PC. Alternatively, it can be installed on both the microscope PC and the camera PC, running `instamatic.temserver` on the microscope PC, and establishing a connection over the local network. See the config documentation for how to set this up. +For FEI microscopes, `instamatic` must be installed on the microscope control PC. Alternatively, it can be installed on both the microscope PC and the camera PC, running `instamatic.temserver` on the microscope PC, and establishing a connection over the local network. If any server PC does not support modern software, [instamatic-dev/instamatic-tecnai-server](https://github.com/instamatic-dev/instamatic-tecnai-server) is a drop-in replacement that requires Python 3.4 only. + +See the config documentation for how to set this up. ## Development version @@ -32,7 +34,7 @@ In order of priority: ### __2. Set up the microscope interface__ Go to the config directory from the first step. - In `config/settings.yaml` define the camera interface you want to use. You can use the autoconfig tool or one of the example files and modify those. You can name these files anything you want, as long as the name under `microscope` matches the filename in `config/microscope` + In `config/settings.yaml` define the camera interface you want to use. You can use the autoconfig tool or one of the example files and modify those. You can name these files anything you want, as long as the name under `microscope` matches the filename in `config/microscope`. ### __3. Set up the magnifications and camera lengths__ In the config file, i.e `config/microscope/jeol.yaml`, set the correct camera lengths (`ranges/diff`) and magnifications for your microscopes (`ranges/lowmag` and `ranges/mag1`). Also make sure you set the wavelength. Again, the autoconfig tool is your best friend, otherwise, the way to get those numbers is to simply write them down as you turn the magnification knob on the microcope. @@ -41,10 +43,10 @@ In order of priority: Specify the file you want to use for the camera interface, i.e. `camera: timepix` points to `config/camera/timepix.yaml`. In this file, make sure that the interface is set to your camera type and update the numbers as specified in the config documentation. If you do not want to set up the camera interface at this moment, you can use `camera: simulate` to fake the camera connection. ### __5. Make the calibration table__ - For each of the magnfications defined in `config/microscope/jeol.yaml`, specify the pixel sizes in the file defined by `calibration: jeol`, corresponding to the file `calibration/jeol.yaml`. For starters, you can simply set the calibration values to 1.0. + For each of the magnifications defined in `config/microscope/jeol.yaml`, specify the pixel sizes in the file defined by `calibration: jeol`, corresponding to the file `calibration/jeol.yaml`. For starters, you can simply set the calibration values to 1.0. ### __6. Test if it works__ - Run `instamatic.temcontroller` to start a IPython shell that initializes the connection. It should run with no crashes or warnings. + Run `instamatic.controller` to start a IPython shell that initializes the connection. It should run with no crashes or warnings. ### __7. Update `settings.yaml`__ There are a few more choices to make in `instamatic/settings.yaml`. If you use a TVIPS camera, make sure you put `use_cam_server: true`. diff --git a/readme.md b/readme.md index c2413c56..7100550f 100644 --- a/readme.md +++ b/readme.md @@ -22,11 +22,12 @@ Cameras supported: - ASI CheeTah through `serval-toolkit` library - TVIPS cameras through EMMENU4 API - Quantum Detectors MerlinEM +- Gatan cameras through FEI scripting interface - (Gatan cameras through DM plugin [1]) Instamatic has been developed on a JEOL-2100 with a Timepix camera, and a JEOL-1400 and JEOL-3200 with TVIPS cameras (XF416/F416). -See [instamatic-dev/instamatic-tecnai-server](https://github.com/instamatic-dev/instamatic-tecnai-server) for a TEM interface to control a FEI Tecnai-TEM on Windows XP/Python 3.4 via instamatic. +See [instamatic-dev/instamatic-tecnai-server](https://github.com/instamatic-dev/instamatic-tecnai-server) for a TEM interface to control a FEI Tecnai or FEI Titan TEM and associated cameras on Windows XP/Python 3.4 via instamatic. [1]: Support for Gatan cameras is somewhat underdeveloped. As an alternative, a DigitalMicrograph script for collecting cRED data on a OneView camera (or any other Gatan camera) can be found [here](https://github.com/instamatic-dev/InsteaDMatic). @@ -47,7 +48,7 @@ pip install instamatic ## OS requirement -The package requires Windows 7 or higher. It has been mainly developed and tested under windows 7 and higher. +The package requires Windows 7 or higher. It has been mainly developed and tested under Windows 7 and higher. ## Package dependencies From 12813c75f8429380c906ad6dfca1356d87a2f20b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Tcho=C5=84?= Date: Tue, 13 Jan 2026 19:44:24 +0100 Subject: [PATCH 13/13] Update `docs/requirements.txt` to prevent ongoing version skew --- docs/requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/requirements.txt b/docs/requirements.txt index 86cf67ba..a479acfe 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -3,4 +3,5 @@ mkdocs mkdocs-jupyter mkdocs-gen-files mkdocs-material -mkdocstrings[python] +mkdocstrings>=0.26 +mkdocstrings-python>=1.10