diff --git a/arkindex/process/api.py b/arkindex/process/api.py
index db75eac89a6d60edf5bb02e1cd62985737cfc2d0..13c1dfebb7cc10ae9e0162b61d92403df012aa44 100644
--- a/arkindex/process/api.py
+++ b/arkindex/process/api.py
@@ -774,7 +774,11 @@ class ProcessDatasetManage(CreateAPIView, UpdateAPIView, DestroyAPIView):
     def destroy(self, request, *args, **kwargs):
         serializer = self.get_serializer(data=request.data)
         serializer.is_valid(raise_exception=True)
-        get_object_or_404(ProcessDataset, **serializer.validated_data).delete()
+        # Ignore the sets when retrieving the ProcessDataset instance, as there cannot be
+        # two ProcessDatasets with the same dataset and process, whatever the sets
+        validated_data = serializer.validated_data
+        del validated_data["sets"]
+        get_object_or_404(ProcessDataset, **validated_data).delete()
         return Response(status=status.HTTP_204_NO_CONTENT)
 
 
diff --git a/arkindex/process/tests/test_process_datasets.py b/arkindex/process/tests/test_process_datasets.py
index 16005a9513b7e9cf840ccdd24c948074440a595d..4efcd2af60da5b055467172f64204d0f2d66f464 100644
--- a/arkindex/process/tests/test_process_datasets.py
+++ b/arkindex/process/tests/test_process_datasets.py
@@ -50,7 +50,6 @@ class TestProcessDatasets(FixtureAPITestCase):
             corpus_id=cls.corpus.id
         )
         ProcessDataset.objects.create(process=cls.dataset_process_2, dataset=cls.dataset2, sets=cls.dataset2.sets)
-        cls.dataset_process_2.datasets.set([cls.dataset2])
 
         # For repository process
         cls.repo = Repository.objects.get(url="http://my_repo.fake/workers/worker")
@@ -766,3 +765,18 @@ class TestProcessDatasets(FixtureAPITestCase):
             )
             self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
         self.assertFalse(ProcessDataset.objects.filter(process=self.dataset_process, dataset=self.dataset1).exists())
+
+    def test_destroy_sets_agnostic(self):
+        """
+        When deleting a process dataset, it doesn't matter what its sets are as there cannot be two process datasets
+        with the same process and dataset, whatever the sets are.
+        """
+        self.process_dataset_1.sets = ["test"]
+        self.process_dataset_1.save()
+        self.client.force_login(self.test_user)
+        with self.assertNumQueries(9):
+            response = self.client.delete(
+                reverse("api:process-dataset", kwargs={"process": self.dataset_process.id, "dataset": self.dataset1.id}),
+            )
+            self.assertEqual(response.status_code, status.HTTP_204_NO_CONTENT)
+        self.assertFalse(ProcessDataset.objects.filter(process=self.dataset_process, dataset=self.dataset1).exists())