Merge pull request #10 from russellb/ilab-sync

Sync from instructlab

Makefile

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
-SDG_IMPORT_REF:=d9e7bf2f59819fcd42d9648b0ebbb81b6d2bf893
+SDG_IMPORT_REF:=e5368f0b0a2d019f6aa56837f9a3c6ee6ba72197
 
 #
 # If you want to see the full commands, run:
@@ -43,7 +43,7 @@ check: ## Check git diff between this repo and the CLI generator directory
 	@echo "==="
 	@echo "=== CHANGES SINCE LAST IMPORT FROM instructlab/instructlab repo:"
 	@echo "==="
-	@git diff $(SDG_IMPORT_REF)..origin/main -- src/instructlab/generator/ | cat
+	@git diff $(SDG_IMPORT_REF)..instructlab_repo/main -- src/instructlab/generator/ | cat
 
 .PHONY: check-tox
 check-tox:

src/instructlab_sdg/generate_data.py

@@ -15,7 +15,7 @@
 
 # Third Party
 # instructlab - All of these need to go away - issue #6
-from instructlab.config import DEFAULT_MULTIPROCESSING_START_METHOD, get_model_family
+from instructlab.config import get_model_family
 from instructlab.utils import (
     chunk_document,
     max_seed_example_tokens,
@@ -493,7 +493,7 @@ def unescape(s):
             "Synthesizing new instructions. If you aren't satisfied with the generated instructions, interrupt training (Ctrl-C) and try adjusting your YAML files. Adding more examples may help."
         )
 
-    mpctx = multiprocessing.get_context(DEFAULT_MULTIPROCESSING_START_METHOD)
+    mpctx = multiprocessing.get_context(None)
 
     all_taxonomy_paths = list(set(e["taxonomy_path"] for e in seed_instruction_data))
     total_discarded = 0