## ----include=FALSE------------------------------------------------------------ knitr::opts_chunk$set( collapse = TRUE, comment = "#>" ) ## ----setup, message=FALSE----------------------------------------------------- library(pairwiseLLM) library(dplyr) library(tidyr) library(purrr) library(readr) library(stringr) ## ----------------------------------------------------------------------------- check_llm_api_keys() ## ----------------------------------------------------------------------------- data("example_writing_samples", package = "pairwiseLLM") td <- trait_description("overall_quality") td ## ----------------------------------------------------------------------------- tmpl <- set_prompt_template() cat(substr(tmpl, 1, 400), "... ") ## ----------------------------------------------------------------------------- set.seed(123) pairs_all <- example_writing_samples |> make_pairs() n_pairs <- min(40L, nrow(pairs_all)) pairs_forward <- pairs_all |> sample_pairs(n_pairs = n_pairs, seed = 123) |> randomize_pair_order(seed = 456) pairs_reverse <- sample_reverse_pairs( pairs_forward, reverse_pct = 1.0, seed = 789 ) get_pairs_for_direction <- function(direction = c("forward", "reverse")) { direction <- match.arg(direction) if (identical(direction, "forward")) { pairs_forward } else { pairs_reverse } } ## ----------------------------------------------------------------------------- anthropic_models <- c( "claude-haiku-4-5-20251001" ) gemini_models <- c( "gemini-3.8-flash" ) openai_models <- c( "gpt-4.1", "gpt-5.6-luna" ) thinking_levels <- c("no_thinking", "with_thinking") directions <- c("forward", "reverse") anthropic_grid <- tidyr::expand_grid( provider = "anthropic", model = anthropic_models, thinking = "no_thinking", direction = directions ) gemini_grid <- tidyr::expand_grid( provider = "gemini", model = gemini_models, thinking = "with_thinking", direction = directions ) openai_grid <- tidyr::expand_grid( provider = "openai", model = openai_models, thinking = "no_thinking", direction = directions ) batch_grid <- dplyr::bind_rows( anthropic_grid, gemini_grid, openai_grid ) batch_grid ## ----------------------------------------------------------------------------- templates_tbl <- tibble::tibble( template_id = c("test1", "test2", "test3", "test4", "test5"), prompt_template = list(tmpl, tmpl, tmpl, tmpl, tmpl) ) templates_tbl ## ----eval=FALSE--------------------------------------------------------------- # out_root <- "dev-output/advanced-multi-batch" # dir.create(out_root, recursive = TRUE, showWarnings = FALSE) # # run_plan <- tidyr::crossing( # templates_tbl |> tidyr::unnest(prompt_template), # batch_grid # ) |> # mutate( # run_id = paste(template_id, provider, model, thinking, direction, sep = "__"), # run_id = gsub("[^A-Za-z0-9_.-]+", "-", run_id), # run_dir = file.path(out_root, run_id) # ) # # run_plan |> dplyr::select(run_id, template_id, provider, model, thinking, direction, run_dir) ## ----eval=FALSE--------------------------------------------------------------- # submit_one_run <- function(template_id, prompt_template, provider, model, thinking, direction, run_dir) { # pairs_use <- get_pairs_for_direction(direction) # is_thinking <- identical(thinking, "with_thinking") # # # Provider-specific knobs (passed through via ...) # extra_args <- list() # # if (identical(provider, "openai")) { # # Only request thoughts for models that support them in this workflow # extra_args$include_thoughts <- is_thinking && grepl("^gpt-5", model) # extra_args$include_raw <- TRUE # } else if (identical(provider, "anthropic")) { # extra_args$reasoning <- if (is_thinking) "enabled" else "none" # extra_args$include_thoughts <- is_thinking # extra_args$include_raw <- TRUE # # Optional: set deterministic temperature when not using reasoning # if (!is_thinking) extra_args$temperature <- 0 # } else if (identical(provider, "gemini")) { # extra_args$include_thoughts <- TRUE # extra_args$thinking_level <- "low" # example # extra_args$include_raw <- TRUE # } # # message( # "Submitting: ", template_id, " | ", provider, " / ", model, # " / ", thinking, " / ", direction # ) # # # Split strategy: # # - For real jobs, use batch_size (e.g., 500–5000) or n_segments (e.g., 10–50) # # - Here we keep it simple and submit a single segment per run # do.call( # llm_submit_pairs_multi_batch, # c( # list( # pairs = pairs_use, # backend = provider, # model = model, # trait_name = td$name, # trait_description = td$description, # prompt_template = prompt_template, # n_segments = 1L, # output_dir = run_dir, # write_registry = TRUE, # verbose = TRUE # ), # extra_args # ) # ) # } # # run_results <- purrr::pmap( # run_plan |> # dplyr::select( # template_id, prompt_template, provider, model, thinking, direction, # run_dir # ), # submit_one_run # ) # # # Store a lightweight manifest so you can resume later without rebuilding run_plan # manifest <- run_plan |> # mutate(registry_path = file.path(run_dir, "jobs_registry.csv")) # # manifest_path <- file.path(out_root, "run_manifest.csv") # readr::write_csv(manifest, manifest_path) # # manifest_path ## ----eval=FALSE--------------------------------------------------------------- # manifest_path <- file.path(out_root, "run_manifest.csv") # manifest <- readr::read_csv(manifest_path, show_col_types = FALSE) # # poll_one_run <- function(run_dir) { # llm_resume_multi_batches( # jobs = NULL, # load from jobs_registry.csv in run_dir # output_dir = run_dir, # interval_seconds = 60, # per_job_delay = 2, # write_results_csv = TRUE, # writes batch_XX_results.csv files # write_registry = TRUE, # refreshes jobs_registry.csv with done flags # keep_jsonl = TRUE, # verbose = TRUE, # write_combined_csv = TRUE, # writes combined_results.csv inside run_dir # combined_csv_path = "combined_results.csv" # ) # } # # polled <- purrr::map(manifest$run_dir, poll_one_run) ## ----eval=FALSE--------------------------------------------------------------- # combined_all <- purrr::map2_dfr( # polled, # seq_len(nrow(manifest)), # function(res, i) { # meta <- manifest[i, ] # if (is.null(res$combined)) return(NULL) # # res$combined |> # mutate( # template_id = meta$template_id, # provider = meta$provider, # model = meta$model, # thinking = meta$thinking, # direction = meta$direction, # run_id = meta$run_id # ) # } # ) # # combined_path <- file.path(out_root, "combined_all_runs.csv") # readr::write_csv(combined_all, combined_path) # # combined_path ## ----eval=FALSE--------------------------------------------------------------- # manifest <- readr::read_csv(file.path(out_root, "run_manifest.csv"), show_col_types = FALSE) # # needs_poll <- function(run_dir) { # reg_path <- file.path(run_dir, "jobs_registry.csv") # if (!file.exists(reg_path)) return(FALSE) # reg <- readr::read_csv(reg_path, show_col_types = FALSE) # any(!as.logical(reg$done)) # } # # unfinished_dirs <- manifest$run_dir[vapply(manifest$run_dir, needs_poll, logical(1))] # # polled <- purrr::map(unfinished_dirs, poll_one_run) .