Skip to content

LIMS-2181: Improve speed of queueing and editing 1000s of samples #1070

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
ndg63276 merged 5 commits into from
May 18, 2026
Merged

LIMS-2181: Improve speed of queueing and editing 1000s of samples #1070

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
484ad27
LIMS-2181: Improve speed of queueing and editing 1000s of samples
May 14, 2026
70a0299
LIMS-2181: Do in chunks of 500
May 15, 2026
5446ecb
LIMS-2181: Remove very slow validation
May 15, 2026
ad73013
Update client/src/js/modules/imaging/views/queuecontainer.js
ndg63276 May 18, 2026
0ba990a
LIMS-2181: Use placeholders rather than directly using ids
May 18, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
107 changes: 103 additions & 4 deletions api/src/Page/Sample.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ class Sample extends Page
array('/sub/:ssid', 'get', '_get_sub_sample'),
array('/sub/:ssid', 'patch', '_update_sub_sample'),
array('/sub/:ssid', 'put', '_update_sub_sample_full'),
array('/sub/cid/:cid', 'post', '_update_container_sub_samples'),
array('/sub', 'post', '_add_sub_sample'),
array('/sub/:ssid', 'delete', '_delete_sub_sample'),
array('/sub/queue/cid/:cid', 'post', '_queue_all_sub_samples'),
Expand Down Expand Up @@ -638,13 +639,49 @@ function _queue_all_sub_samples()

$this->db->wait_rep_sync(false);

if (!sizeof($subs))
$this->_error('No subsamples found');

$sub_ids = array_column($subs, 'BLSUBSAMPLEID');
$chunks = array_chunk($sub_ids, 500);
$ret = array();
foreach ($subs as $sub) {
array_push($ret, array(
'BLSUBSAMPLEID' => $sub['BLSUBSAMPLEID'],
'CONTAINERQUEUESAMPLEID' => $this->_do_pre_q_sample(array('BLSUBSAMPLEID' => $sub['BLSUBSAMPLEID']))));

if ($this->has_arg('UNQUEUE')) {
foreach ($chunks as $chunk) {
$placeholders = implode(',', array_fill(0, count($chunk), '?'));
$this->db->pq("DELETE FROM containerqueuesample
WHERE containerqueueid IS NULL
AND blsubsampleid IN ($placeholders)", $chunk);
}

$ret = array_map(function($id) {
return array('BLSUBSAMPLEID' => $id, 'CONTAINERQUEUESAMPLEID' => null);
}, $sub_ids);

} else {
foreach ($chunks as $chunk) {
$placeholders = implode(',', array_fill(0, count($chunk), '?'));
$this->db->pq("INSERT INTO containerqueuesample (blsubsampleid)
SELECT ss.blsubsampleid
FROM blsubsample ss
INNER JOIN blsample s ON s.blsampleid = ss.blsampleid
INNER JOIN container c ON c.containerid = s.containerid
INNER JOIN dewar d ON d.dewarid = c.dewarid
INNER JOIN shipping sh ON sh.shippingid = d.shippingid
WHERE sh.proposalid = ?
AND ss.blsubsampleid IN ($placeholders)",
array_merge(array($this->proposalid), $chunk));

$chunk_ret = $this->db->pq("SELECT blsubsampleid, containerqueuesampleid
FROM containerqueuesample
WHERE blsubsampleid IN ($placeholders)
AND containerqueueid IS NULL", $chunk);

$ret = array_merge($ret, $chunk_ret);
}
}
$this->_output($ret);

}

function _sub_samples()
Expand Down Expand Up @@ -960,6 +997,68 @@ function _update_sub_sample_full()
}


function _update_container_sub_samples()
{
if (!$this->arg('cid'))
$this->_error('No container specified');

if (!$this->arg('EXPERIMENTKIND'))
$this->_error('No experiment kind provided');

$rows = $this->db->pq("SELECT DISTINCT ss.diffractionplanid
FROM blsubsample ss
INNER JOIN blsample s ON s.blsampleid = ss.blsampleid
INNER JOIN diffractionplan dp ON dp.diffractionplanid = ss.diffractionplanid
INNER JOIN containerqueuesample cqs ON cqs.blsubsampleid = ss.blsubsampleid
WHERE s.containerid=:1
AND dp.experimentkind=:2
AND ss.diffractionplanid IS NOT NULL
AND cqs.containerqueuesampleid IS NOT NULL
AND cqs.containerqueueid IS NULL",
array($this->arg('cid'), $this->arg('EXPERIMENTKIND')));

if (!sizeof($rows)) {
$this->_output(array('TOTAL_UPDATED' => 0));
return;
}

$all_ids = array_column($rows, 'DIFFRACTIONPLANID');
$set_args = array();
foreach (array(
'REQUIREDRESOLUTION', 'EXPERIMENTKIND', 'PREFERREDBEAMSIZEX', 'PREFERREDBEAMSIZEY',
'EXPOSURETIME', 'BOXSIZEX', 'BOXSIZEY', 'AXISSTART', 'AXISRANGE', 'NUMBEROFIMAGES',
'TRANSMISSION', 'ENERGY', 'MONOCHROMATOR'
) as $f) {
array_push($set_args, $this->has_arg($f) ? $this->arg($f) : null);
}

$chunks = array_chunk($all_ids, 500);
$total_updated = 0;

foreach ($chunks as $chunk) {
$start_index = sizeof($set_args) + 1;
$placeholders = array();

foreach ($chunk as $index => $id) {
array_push($placeholders, ':' . ($start_index + $index));
}

$id_placeholders_list = implode(',', $placeholders);

$combined_args = array_merge($set_args, $chunk);

$this->db->pq("UPDATE diffractionplan
SET requiredresolution=:1, experimentkind=:2, preferredbeamsizex=:3, preferredbeamsizey=:4,
exposuretime=:5, boxsizex=:6, boxsizey=:7, axisstart=:8, axisrange=:9,
numberofimages=:10, transmission=:11, energy=:12, monochromator=:13
WHERE diffractionplanid IN ($id_placeholders_list)", $combined_args);

$total_updated += sizeof($chunk);
}

$this->_output(array('TOTAL_UPDATED' => $total_updated));
}


function _add_sub_sample()
{
Expand Down
55 changes: 29 additions & 26 deletions client/src/js/modules/imaging/views/queuecontainer.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -683,7 +683,7 @@ define(['marionette',
app.message({ message: 'Container Successfully Unqueued' })
self.ui.unqueuebutton.hide()
self.ui.queuebutton.show()
self.model.set('CONTAINERQUEUEID', null)
self.model.set('CONTAINERQUEUEID', null, { silent: true })
},
error: function() {
app.alert({ message: 'Something went wrong unqueuing this container' })
Expand All @@ -704,23 +704,36 @@ define(['marionette',

var p = this.plans.findWhere({ DIFFRACTIONPLANID: this.ui.preset.val() })
if (p) {
self.ui.applyall.html('Applying...').prop('disabled', true)
var promises = await this.applyModel(p, false)

const updateButtonAfterProcessing = () => {
self.ui.applyall.html('<i class="fa fa-file-text-o"></i> Apply to All').prop('disabled', false)
var updateParams = {
EXPERIMENTKIND: p.get('EXPERIMENTKIND')
}

if (promises && promises.length > 0) {
Promise.allSettled(promises)
.then(updateButtonAfterProcessing)
.catch(error => {
console.error("Error after promises settled: ", error);
updateButtonAfterProcessing()
})
} else {
updateButtonAfterProcessing()
}
const fields = [
'REQUIREDRESOLUTION', 'PREFERREDBEAMSIZEX', 'PREFERREDBEAMSIZEY',
'EXPOSURETIME', 'BOXSIZEX', 'BOXSIZEY', 'AXISSTART', 'AXISRANGE',
'NUMBEROFIMAGES', 'TRANSMISSION', 'ENERGY', 'MONOCHROMATOR'
]

fields.forEach(function(k) {
if (p.get(k) !== null && p.get(k) !== undefined) {
updateParams[k] = p.get(k)
}
})

Backbone.ajax({
url: app.apiurl+'/sample/sub/cid/'+this.model.get('CONTAINERID'),
method: 'POST',
data: updateParams,
success: function(json) {
self.refreshSubSamples()
app.message({ message: 'Applied preset to ' + json.TOTAL_UPDATED + ' samples' })
},
error: function() {
app.alert({ message: 'Something went wrong applying to all' })
}
})

}
},

Expand Down Expand Up @@ -777,17 +790,7 @@ define(['marionette',
return
}

// need to validate all models here again in case they haven't been rendered
this.qsubsamples.fullCollection.each(function(qs) {
if (qs.get('_valid') !== undefined) return

const expcell = new ExperimentCell({model: qs, column: {beamlinesetups: this.beamlinesetups}});
const val = expcell.checkIsValid();
console.log({ experimentCell: val })
}, this)

var invalid = this.typeselector.shadowCollection.where({ '_valid': false })
console.log('queue', invalid, invalid.length > 0)
if (invalid.length > 0) {
app.alert({ message: 'There are '+invalid.length+' sub samples with invalid experimental plans, please either correct or remove these from the queue' })
var inv = this.typeselector.collection.findWhere({ id: 'invalid' })
Expand All @@ -804,7 +807,7 @@ define(['marionette',
app.message({ message: 'Container Successfully Queued' })
self.ui.unqueuebutton.show()
self.ui.queuebutton.hide()
self.model.set('CONTAINERQUEUEID', json.CONTAINERQUEUEID)
self.model.set('CONTAINERQUEUEID', json.CONTAINERQUEUEID, { silent: true })
},
error: function() {
app.alert({ message: 'Something went wrong queuing this container' })
Expand Down
Loading