models:
- name: "ar+nar"
  size: "full"
  resp_levels: 8
  prom_levels: 8
  tasks: 8
  langs: 2
  tones: 1
  arch_type: "retnet"
  training: True
  version: 3

hyperparameters:
  batch_size: 4
  gradient_accumulation_steps: 4
  gradient_clipping: 10
  
  optimizer: Adagrad
  torch_optimizer: True
  learning_rate: 1.0e-2
  
  scheduler_type: ""
  #scheduler_type: OneCycle
  #scheduler_params:
  #  cycle_first_step_size: 10_000
  #  cycle_first_stair_count: 10_000

  #  cycle_second_step_size: 15_000
  #  cycle_second_stair_count: 15_000

  #  decay_step_size: 5_000

  #  cycle_min_lr: 2.5e-4 # 1.0e-5
  #  cycle_max_lr: 2.5e-4 # 1.0e-4
  #  decay_lr_rate: 0.0

  #  cycle_min_mom: 0.90
  #  cycle_max_mom: 0.99
  #  decay_mom_rate: 0.0

evaluation:
  batch_size: 8
  frequency: 10000
  size: 8
  
  steps: 500
  ar_temperature: 0.95
  nar_temperature: 0.25
  load_disabled_engines: True

trainer:
  no_logger: True

  iterations: 1_000_000
  
  save_tag: step
  save_on_oom: True
  save_on_quit: True
  save_frequency: 250
  export_on_save: True

  keep_last_checkpoints: 4

  aggressive_optimizations: False
  load_disabled_engines: False

  #load_state_dict: True
  strict_loading: False
  #load_tag: "9500"
  #load_states: False
  #restart_step_count: True
  
  gc_mode: None # "global_step"

  weight_dtype: float32
  amp: False

  backend: deepspeed
  deepspeed:
    inferencing: True
    zero_optimization_level: 0
    use_compression_training: False

  activation_checkpointing: True

  load_webui: True

inference:
  backend: deepspeed
  audio_backend: "dac"
  normalize: False

  weight_dtype: float32
  amp: False

bitsandbytes:
  enabled: False

  injects: False
  replace: False

  linear: False
  embedding: False
  
  bitnet: False

fp8:
  enabled: False
  backend: "te"
  
experimental: True

dataset:
  speaker_name_getter: "lambda p: f'{p.parts[-3]}_{p.parts[-2]}'"
  speaker_group_getter: "lambda p: f'{p.parts[-3]}'"
  speaker_languages:
    ja: []

  use_hdf5: True
  use_metadata: True
  hdf5_flag: r
  validate: True

  workers: 8
  cache: True

  #phones_range: [4, 512]
  #duration_range: [1.0, 32.0]
  
  phones_range: [0, 512]
  duration_range: [0.0, 64.0]

  random_utterance: 1.0
  max_prompts: 3
  prompt_duration: 6.0
  
  max_resps: 1
  p_resp_append: 0.25

  sample_type: speaker

  tasks_list: [ "tts" ] # , [ "tts", "tts-c", "ns", "sr", "tse", "cse", "nse", "tts"]

  training: []
  validation: []
  noise: []