pulsatrix 1.1.0
Loading...
Searching...
No Matches
lora.hpp
Go to the documentation of this file.
1
17#pragma once
18
19#include <cstdint>
20#include <string>
21#include <utility>
22#include <vector>
23
26#include "pulsatrix/module.hpp"
27#include "pulsatrix/tensor.hpp"
28
29namespace pulsatrix {
30
31class LinearModule;
32
34struct LoRAConfig {
36 int64_t rank = 8;
38 float alpha = 8.0f;
40 float dropout = 0.0f;
42 uint64_t seed = 0;
49 std::vector<std::string> target_modules;
50};
51
59public:
65 LoRAAdapter(int64_t in_features, int64_t out_features, const LoRAConfig& config, DeviceBackend* backend,
66 DeviceType device);
67
68 [[nodiscard]] int64_t rank() const { return rank_; }
69 [[nodiscard]] float alpha() const { return alpha_; }
71 [[nodiscard]] float scale() const { return alpha_ / static_cast<float>(rank_); }
72 [[nodiscard]] float dropout() const { return dropout_p_; }
73 [[nodiscard]] const Tensor& a() const { return a_; }
74 [[nodiscard]] const Tensor& b() const { return b_; }
75 [[nodiscard]] const Tensor& a_grad() const { return a_grad_; }
76 [[nodiscard]] const Tensor& b_grad() const { return b_grad_; }
78 void set_a(const std::vector<float>& values);
80 void set_b(const std::vector<float>& values);
81
83 [[nodiscard]] bool merged() const { return merged_; }
85 [[nodiscard]] bool enabled() const { return enabled_; }
86
88 [[nodiscard]] Tensor effective_weight(const Tensor& weight) const;
90 [[nodiscard]] Tensor update() const;
91
96 [[nodiscard]] bool has_relevance() const { return last_adapter_relevance_.numel() > 0; }
99 [[nodiscard]] const Tensor& last_base_relevance() const;
103 [[nodiscard]] const Tensor& last_adapter_relevance() const;
104
105private:
106 friend class LinearModule;
108 void add_forward(const Tensor& input, Tensor& pre_bias, bool training);
110 void backward(const Tensor& grad_output, Tensor& grad_input);
112 void add_update(Tensor& weight, float sign) const;
113 void release_activations();
114 void clear_relevance();
115
116 int64_t in_features_;
117 int64_t out_features_;
118 int64_t rank_;
119 float alpha_;
120 float dropout_p_;
121 DeviceBackend* backend_;
122 Tensor a_;
123 Tensor b_;
124 Tensor a_grad_;
125 Tensor b_grad_;
126 DropoutModule dropout_;
127 Tensor last_input_; // the adapter's input, after dropout
128 Tensor last_hidden_; // scale() x A, (N, r)
129 Tensor last_base_relevance_;
130 Tensor last_adapter_relevance_;
131 bool used_dropout_ = false;
132 bool merged_ = false;
133 bool enabled_ = true;
134};
135
145std::vector<std::string> ApplyLora(Module& model, const LoRAConfig& config = {});
146
148[[nodiscard]] std::vector<std::pair<std::string, LinearModule*>> LoraLayers(Module& model);
149
153 std::string name;
159 double base_total = 0;
160 double adapter_total = 0;
161};
162
171[[nodiscard]] std::vector<LoraLayerRelevance> LoraRelevanceSplit(Module& model);
172
174void MergeLora(Module& model);
176void UnmergeLora(Module& model);
178void SetLoraEnabled(Module& model, bool enabled);
183void RemoveLora(Module& model);
184
199void SaveLoraAdapter(Module& model, const std::string& directory, const std::vector<WeightMapping>& mapping = {});
200
212void LoadLoraAdapter(Module& model, const std::string& directory, const std::vector<WeightMapping>& mapping = {});
213
214} // namespace pulsatrix
A Llama-family decoder-only language model built from a Hugging Face config, and loading its Hugging ...
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
y = (mask_i ? x_i / (1 - p) : 0) at training time (inverted dropout – scaling happens at training tim...
Definition dropout_module.hpp:30
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
One layer's low-rank update: A (in, r) and B (r, out), stored like LinearModule's weight,...
Definition lora.hpp:58
bool has_relevance() const
True after a relevance pass that split this layer (LoRARelevance::Split, adapter enabled and unmerged...
Definition lora.hpp:96
const Tensor & last_base_relevance() const
The last split's input relevance through the base weight W, (N, in).
const Tensor & b_grad() const
Definition lora.hpp:76
const Tensor & a_grad() const
Definition lora.hpp:75
float dropout() const
Definition lora.hpp:72
const Tensor & a() const
Definition lora.hpp:73
void set_a(const std::vector< float > &values)
Replaces A, (in, r) row-major.
int64_t rank() const
Definition lora.hpp:68
bool enabled() const
False while the layer runs as its base layer (LinearModule::set_lora_enabled()).
Definition lora.hpp:85
float alpha() const
Definition lora.hpp:69
const Tensor & b() const
Definition lora.hpp:74
Tensor effective_weight(const Tensor &weight) const
The adapted weight W + scale() A B, as a new tensor.
Tensor update() const
The update scale() A B alone, (in, out).
LoRAAdapter(int64_t in_features, int64_t out_features, const LoRAConfig &config, DeviceBackend *backend, DeviceType device)
A as PEFT initializes it (Kaiming-uniform with a = sqrt(5), so uniform on ±1 / sqrt(in)),...
float scale() const
alpha / rank.
Definition lora.hpp:71
bool merged() const
True while the update is added into the base weight (LinearModule::merge_lora()).
Definition lora.hpp:83
void set_b(const std::vector< float > &values)
Replaces B, (r, out) row-major.
const Tensor & last_adapter_relevance() const
The last split's input relevance through the adapter, (N, in): what the fine-tuning contributes to th...
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
int64_t numel() const
Total element count – shape().numel().
Definition tensor.hpp:116
Inverted dropout – the first module whose forward behavior genuinely differs between training and inf...
Abstract base every layer subclasses – NVI forward(), pure-virtual LRP contract.
Definition acquisition_functions.hpp:16
void MergeLora(Module &model)
Merges every adapter into its base weight (LinearModule::merge_lora()).
void LoadLoraAdapter(Module &model, const std::string &directory, const std::vector< WeightMapping > &mapping={})
Reads a PEFT LoRA adapter directory onto the model: attaches an adapter to each layer it has weights ...
void RemoveLora(Module &model)
Removes every adapter. Merged ones stay in the weights (PEFT's merge_and_unload() is MergeLora() then...
std::vector< std::pair< std::string, LinearModule * > > LoraLayers(Module &model)
The model's LinearModules that have an adapter, by name, in the model's order.
DeviceType
Which physical device a Tensor's buffer resides on.
Definition device_backend.hpp:17
void SetLoraEnabled(Module &model, bool enabled)
Turns every adapter on or off, so one model runs as the base or as the fine-tune.
void SaveLoraAdapter(Module &model, const std::string &directory, const std::vector< WeightMapping > &mapping={})
Writes the model's adapters as a PEFT adapter directory: adapter_model.safetensors and adapter_config...
void UnmergeLora(Module &model)
Takes every merged adapter out of its base weight again.
std::vector< std::string > ApplyLora(Module &model, const LoRAConfig &config={})
Attaches a LoRA adapter to each LinearModule that config.target_modules selects, then freezes everyth...
std::vector< LoraLayerRelevance > LoraRelevanceSplit(Module &model)
How much of each adapted layer's relevance went through its adapter in the model's last relevance pas...
LoRA's settings. The defaults are PEFT's LoraConfig's.
Definition lora.hpp:34
int64_t rank
The rank r of the update.
Definition lora.hpp:36
std::vector< std::string > target_modules
The layers ApplyLora() adapts, PEFT style: a layer matches a name if its own name is that name or end...
Definition lora.hpp:49
float alpha
The update is scaled by alpha / rank.
Definition lora.hpp:38
uint64_t seed
Seeds A's initialization (and the dropout masks), the same on every platform.
Definition lora.hpp:42
float dropout
Dropout on the adapter's input, in training mode only.
Definition lora.hpp:40
One adapted layer's share of a relevance pass (TRN-8).
Definition lora.hpp:151
double base_total
Sums over everything: the layer's relevance carried by each path.
Definition lora.hpp:159
double adapter_total
Definition lora.hpp:160
Tensor adapter
Input relevance through the adapter, (N, in).
Definition lora.hpp:157
Tensor base
Input relevance through the base weight, (N, in).
Definition lora.hpp:155
std::string name
The layer's name, as LoraLayers() gives it.
Definition lora.hpp:153
N-dimensional tensor – owns a buffer via DeviceBackend*, RAII (Rule of Five).