pulsatrix 1.1.0
Loading...
Searching...
No Matches
protein_training.hpp
Go to the documentation of this file.
1
7#pragma once
8
9#include <cstdint>
10#include <string>
11#include <vector>
12
17
18namespace pulsatrix {
19
23 double mask_probability = 0.15;
25 double mask_fraction = 0.8;
27 double random_fraction = 0.1;
30 int64_t max_tokens = 1024;
31};
32
44
55public:
59 MaskedLMCollator(const TextTokenizer& tokenizer, DeviceBackend* backend, MaskedLMOptions options = {}, uint64_t seed = 0);
60
63 [[nodiscard]] MaskedLMBatch collate(const std::vector<std::string>& sequences);
64
66 [[nodiscard]] std::vector<int64_t> tokens(const std::string& sequence);
67
68private:
69 const TextTokenizer& tokenizer_;
70 DeviceBackend* backend_;
71 MaskedLMOptions options_;
72 uint64_t state_;
73 int64_t cls_ = 0, eos_ = 0, pad_ = 0, mask_ = 0;
74 std::vector<int64_t> amino_acids_;
75 uint64_t Next();
76 double Uniform();
77};
78
85public:
87 ClusterSampler(const std::vector<std::string>& cluster_of, uint64_t seed = 0);
89 [[nodiscard]] int64_t next();
90 [[nodiscard]] int64_t num_clusters() const { return static_cast<int64_t>(members_.size()); }
91
92private:
93 std::vector<std::vector<int64_t>> members_;
94 uint64_t state_;
95};
96
102void InitializeEsm(EncoderLM& model, uint64_t seed, float std = 0.02f);
103
111float MaskedLMForwardBackward(EncoderLM& model, const MaskedLMBatch& batch, TokenCrossEntropyLoss& loss, float normalizer = 0.0f);
112
115 double loss = 0;
116 double perplexity = 0;
117 double accuracy = 0;
118 int64_t tokens = 0;
119};
120
126[[nodiscard]] MaskedLMEvaluation EvaluateMaskedLM(EncoderLM& model, const TextTokenizer& tokenizer, DeviceBackend* backend,
127 const std::vector<std::string>& sequences, int64_t batch_size = 8,
128 uint64_t seed = 0, MaskedLMOptions options = {});
129
140class SequenceHead : public Module {
141public:
142 enum class Pooling { Mean, PerResidue };
143
146 SequenceHead(int64_t hidden_size, int64_t outputs, Pooling pooling, DeviceBackend* backend, uint64_t seed = 0);
147
150 void set_residue_mask(const Tensor& residues);
151
152 [[nodiscard]] Tensor backward(const Tensor& grad_output) override;
155 [[nodiscard]] Tensor propagate_relevance(const Tensor& relevance_out, const LRPRuleConfig& config) override;
156 [[nodiscard]] OpType op_type() const override { return OpType::Composite; }
157 [[nodiscard]] std::vector<NamedParamRef> named_parameters() override;
158 void set_training(bool training) override;
159 [[nodiscard]] std::optional<DeviceType> compute_device() const override { return backend_->device(); }
160 void release_activations() override;
161
162 [[nodiscard]] LinearModule& linear() { return linear_; }
163 [[nodiscard]] Pooling pooling() const { return pooling_; }
164
165protected:
168 [[nodiscard]] Tensor forward_impl(const Tensor& input) override;
169
170private:
171 int64_t hidden_, outputs_;
172 Pooling pooling_;
173 DeviceBackend* backend_;
174 LinearModule linear_;
175 std::vector<float> residues_;
176 Shape residues_shape_ = Shape({0});
177 std::vector<float> last_input_;
178 Shape last_shape_ = Shape({0});
179};
180
183[[nodiscard]] Tensor ResidueMask(const Tensor& ids, const Tensor* keep, const TextTokenizer& tokenizer, DeviceBackend* backend);
184
185} // namespace pulsatrix
Cluster-weighted sampling, as ESM-2 drew UniRef50: a cluster uniformly, then one of its members unifo...
Definition protein_training.hpp:84
ClusterSampler(const std::vector< std::string > &cluster_of, uint64_t seed=0)
int64_t num_clusters() const
Definition protein_training.hpp:90
int64_t next()
The next sequence's index.
Vendor-agnostic compute/memory backend. CPUBackend, CUDABackend (Phase 1.5), and HIPBackend (Phase 1....
Definition device_backend.hpp:248
virtual DeviceType device() const noexcept=0
Which device this backend's buffers reside on.
A masked language model: token embeddings (with ESM's token-dropout scaling, when the config has it),...
Definition encoder_lm.hpp:109
y = x @ W + b, batched (x is (N, in_features), y is (N, out_features)) – migrated from the original u...
Definition linear_module.hpp:43
Turns sequences into masked-LM batches as ESM-2 was trained: <cls> + residues + <eos>,...
Definition protein_training.hpp:54
MaskedLMBatch collate(const std::vector< std::string > &sequences)
std::vector< int64_t > tokens(const std::string &sequence)
<cls> + sequence (cropped as collate() would) + <eos>, unmasked.
MaskedLMCollator(const TextTokenizer &tokenizer, DeviceBackend *backend, MaskedLMOptions options={}, uint64_t seed=0)
Base class for every layer type (LinearModule, Conv2DModule, activations, ...).
Definition module.hpp:78
A head on an encoder's representations (last_hidden_state), for fine-tuning or probing....
Definition protein_training.hpp:140
Tensor forward_impl(const Tensor &input) override
Tensor backward(const Tensor &grad_output) override
Computes the gradient w.r.t. this module's input, given the gradient w.r.t. its output....
Pooling pooling() const
Definition protein_training.hpp:163
Tensor propagate_relevance(const Tensor &relevance_out, const LRPRuleConfig &config) override
Relevance at the representations, (N, L, hidden): the linear layer's rule, then (for Mean) each resid...
LinearModule & linear()
Definition protein_training.hpp:162
void release_activations() override
Frees the tensors forward() kept for backward() and the relevance passes; containers release their ch...
void set_residue_mask(const Tensor &residues)
Which tokens Pooling::Mean averages: (N, L), 1 for a residue. Set it before every forward() whose sha...
std::optional< DeviceType > compute_device() const override
The device this module computes on, so forward() can reject an input on another device before any ker...
Definition protein_training.hpp:159
std::vector< NamedParamRef > named_parameters() override
This module's trainable parameters, each with its hierarchical name – the one place a module declares...
SequenceHead(int64_t hidden_size, int64_t outputs, Pooling pooling, DeviceBackend *backend, uint64_t seed=0)
The linear layer starts as transformers' heads do: weights from N(0, 0.02²), drawn from seed,...
void set_training(bool training) override
Sets this module's training/eval mode. Defaults to training (matches every mainstream framework's Mod...
OpType op_type() const override
This module's operation-category tag, for ComputationGraph node tagging.
Definition protein_training.hpp:156
Pooling
Definition protein_training.hpp:142
An N-dimensional shape. A plain aggregate of dimensions with no invariant beyond "non-negative dimens...
Definition shape.hpp:24
N-dimensional tensor. Owns its data buffer exclusively; a DeviceBackend* is injected (not owned) – th...
Definition tensor.hpp:29
A tokenizer: added tokens are split out of the raw text first (longest match, earliest first); each r...
Definition text_tokenizer.hpp:166
sum over tokens of -log softmax(logits)[target], divided by a normalizer; tokens whose target is the ...
Definition token_cross_entropy_loss.hpp:28
Masked-language-model encoders: ESM-2 protein language models (PLM-2), Hugging Face EsmForMaskedLM ch...
Dense/fully-connected layer – the reference Module implementation.
Definition acquisition_functions.hpp:16
MaskedLMEvaluation EvaluateMaskedLM(EncoderLM &model, const TextTokenizer &tokenizer, DeviceBackend *backend, const std::vector< std::string > &sequences, int64_t batch_size=8, uint64_t seed=0, MaskedLMOptions options={})
Masks sequences with a collator seeded by seed (so every model sees the same masks) and measures the ...
Tensor ResidueMask(const Tensor &ids, const Tensor *keep, const TextTokenizer &tokenizer, DeviceBackend *backend)
The residue mask for SequenceHead::Pooling::Mean from token ids and a padding mask (or none): 1 for e...
float MaskedLMForwardBackward(EncoderLM &model, const MaskedLMBatch &batch, TokenCrossEntropyLoss &loss, float normalizer=0.0f)
One masked-LM forward and backward: sets the batch's padding mask, runs the model,...
OpType
The op-type tag a Node carries. Charter Part 2 §3: nodes are tagged by a small closed set of op types...
Definition op_type.hpp:19
void InitializeEsm(EncoderLM &model, uint64_t seed, float std=0.02f)
Initializes a model for training from scratch as transformers' EsmPreTrainedModel does: linear weight...
Configuration for LRP relevance propagation: which rule a module applies and its hyperparameters....
Definition lrp_rule_config.hpp:72
One collated batch, on the collator's backend.
Definition protein_training.hpp:34
int64_t num_targets
Picked tokens: the loss's normalizer.
Definition protein_training.hpp:42
Tensor ids
(N, L) token ids: masked inputs, padded with <pad>.
Definition protein_training.hpp:36
Tensor targets
(N, L): the original token where one was picked, -100 (ignored) elsewhere.
Definition protein_training.hpp:40
Tensor keep
(N, L): 1 for a real token, 0 for padding (EncoderLM::set_padding_mask).
Definition protein_training.hpp:38
Masked-LM quality on held-out sequences.
Definition protein_training.hpp:114
int64_t tokens
Definition protein_training.hpp:118
double loss
mean cross-entropy over picked tokens, in nats
Definition protein_training.hpp:115
double accuracy
share of picked tokens whose argmax is the original
Definition protein_training.hpp:117
double perplexity
exp(loss)
Definition protein_training.hpp:116
ESM-2's masking (Lin et al. 2023; BERT's 80/10/10).
Definition protein_training.hpp:21
double mask_fraction
Of the picked residues, the share replaced by <mask>...
Definition protein_training.hpp:25
int64_t max_tokens
The longest a sequence may be, in tokens with <cls> and <eos> (ESM-2's 1024): longer ones are cropped...
Definition protein_training.hpp:30
double random_fraction
...and the share replaced by a random amino acid; the rest stay as they are.
Definition protein_training.hpp:27
double mask_probability
Each residue is picked for prediction with this probability.
Definition protein_training.hpp:23
The tokenizer interface (TOK-1): a pipeline of normalizer, pre-tokenizer, model, post-processor and d...
Cross-entropy over a batch of tokens, with padding ignored and a caller-chosen normalizer for exact g...