https://github.com/JosephSBoyle/skip_gram/blob/346f79ff948ef3d279558a9460c44e7f7598fb7d/skip_gram/main.py

Skip to content Toggle navigation
 
Sign up

  * Product
      +  
        Actions
        Automate any workflow
      +  
        Packages
        Host and manage packages
      +  
        Security
        Find and fix vulnerabilities
      +  
        Codespaces
        Instant dev environments
      +  
        Copilot
        Write better code with AI
      +  
        Code review
        Manage code changes
      +  
        Issues
        Plan and track work
      +  
        Discussions
        Collaborate outside of code
    Explore
      + All features
      + Documentation
      + GitHub Skills
      + Blog
  * Solutions
    For
      + Enterprise
      + Teams
      + Startups
      + Education
    By Solution
      + CI/CD & Automation
      + DevOps
      + DevSecOps
    Case Studies
      + Customer Stories
      + Resources
  * Open Source
      +  
        GitHub Sponsors
        Fund open source developers
      +  
        The ReadME Project
        GitHub community articles
    Repositories
      + Topics
      + Trending
      + Collections
  * Pricing

[                    ] 

  *  
    #
    In this repository All GitHub |
    Jump to |

  * No suggested jump to results

  *  
    #
    In this repository All GitHub |
    Jump to |
  *  
    #
    In this user All GitHub |
    Jump to |
  *  
    #
    In this repository All GitHub |
    Jump to |

Sign in
Sign up
{{ message }}
JosephSBoyle / skip_gram Public

  * Notifications
  * Fork 0
  * Star 21

  * Code
  * Issues 0
  * Pull requests 0
  * Actions
  * Projects 0
  * Security
  * Insights

More

  * Code
  * Issues
  * Pull requests
  * Actions
  * Projects
  * Security
  * Insights

Permalink
346f79ff94
Switch branches/tags
[                    ]
Branches Tags
Could not load branches
Nothing to show
{{ refName }} default View all branches
Could not load tags
Nothing to show
{{ refName }} default
View all tags

Name already in use

A tag already exists with the provided branch name. Many Git commands
accept both tag and branch names, so creating this branch may cause
unexpected behavior. Are you sure you want to create this branch?
Cancel Create

skip_gram/skip_gram/main.py / Jump to
Code definitions

No definitions found in this file.

Code navigation not available for this commit

Go to file

  * Go to file T
  * Go to line L
  * Go to definition R
  * 
  * Copy path
  * Copy permalink

This commit does not belong to any branch on this repository, and may
belong to a fork outside of the repository.
@JosephSBoyle
JosephSBoyle Add more detailed comments + cleanup
Latest commit 346f79f Jun 1, 2023 History
1 contributor

Users who have contributed to this file

184 lines (146 sloc) 7.54 KB
Raw Blame
Edit this file
E
Open in GitHub Desktop

  * 
  * View raw
  * 
  * View blame

This file contains bidirectional Unicode text that may be interpreted
or compiled differently than what appears below. To review, open the
file in an editor that reveals hidden Unicode characters. Learn more
about bidirectional Unicode characters
Show hidden characters

 """Skip-gram (word2vec) training on Spanish text.
 
 Notes:
 - 2 embeddings per-word: one when word is a context and one for when
 the word is a target.
 - We can represent words as the sum of these two vectors, or simply
 throw away the context vector
 and use the target one
 - The dot product between two word vectors can be thought of
 intuitively as the similarity between
 two words. Cosine similarity can be considered a simplified version
 of this 'true' similarity.
 - More out of context ('negative') samples are used during training
 than in-context ones
 - The selection of negative samples has been slightly simplified to
 be unweighted.
 """
 from pathlib import Path
 from pprint import pprint
 import numpy as np
 import random; random.seed(7777)
 ### Utility ###
 def _sigmoid(x: float | np.ndarray) -> float | np.ndarray:
 """Map x to a probability space [0, 1]"""
 return 1 / (1 + np.exp(-x))
 def _cosine_similarity(x1: np.ndarray, x2: np.ndarray) -> float:
 return np.inner(x1, x2) / (np.linalg.norm(x1) * np.linalg.norm(x2))
 def _load_spanish_billion_word_corpus_single_file(
 raw_corpus : list[list[str]],
 min_frequency : int
 ) -> set[str]:
 # 1. Read all the files and construct a vocab of those with
 `min_frequency` or
 # more occurences.
 word_bag: dict[str, int] = dict() # Multiset
 for line in raw_corpus:
 words = line.split()
 for word in words:
 if word in word_bag:
 word_bag[word] += 1
 else:
 word_bag[word] = 1
 keys_to_drop = []
 for k, v in word_bag.items():
 if v < min_frequency:
 keys_to_drop.append(k)
 for k in keys_to_drop:
 del word_bag[k]
 pprint(word_bag)
 print(len(word_bag))
 vocabulary = word_bag.keys()
 return set(vocabulary)
 class EmbeddingDict(dict):
 """Utility wrapper around a regular dict that allows embedding
 lookup by word."""
 def __init__(self, embedding_matrix: np.ndarray, word_to_idx: dict[
 str, int]):
 self._embedding_matrix = embedding_matrix
 self._word_to_idx = word_to_idx
 def __getitem__(self, __key) -> np.ndarray:
 """Get a word's vector representation."""
 return self._embedding_matrix[self._word_to_idx[__key]]
 ### Training functions ###
 def train_skip_gram(
 file : Path,
 min_frequency : int = 100,
 window_size : int = 5,
 alpha : int = 7,
 embedding_dim : int = 100,
 e : float = 1e-3,
 ) -> None:
 """Train a word-to-vector dictionary using the Skip-gram algorithm
 from Mikolov et. al."""
 corpus: list[list[str]] = []
 with open(file, "r", encoding="utf-8") as f:
 raw_corpus: list[list[str]] = f.readlines()
 vocabulary = _load_spanish_billion_word_corpus_single_file(
 raw_corpus, min_frequency)
 for raw_line in raw_corpus:
 line = [word for word in raw_line.split() if word in vocabulary]
 corpus.append(line)
 vocab_count = len(vocabulary)
 vocabulary = list(vocabulary) # Convert to an ordered collection
 from a set.
 word_to_idx = {word : i for i, word in enumerate(vocabulary)}
 ### Randomly initialize word vectors ###
 W = np.random.standard_normal(embedding_dim*vocab_count).reshape((
 embedding_dim, vocab_count))
 """Target vectors."""
 C = np.random.standard_normal(embedding_dim*vocab_count).reshape((
 embedding_dim, vocab_count))
 """Context vectors."""
 ### Training ###
 #
 # In each step, `j`, we're optimizing a logistic regression
 classifier to discriminate
 # between words which do and don't occur around a target word in the
 corpus.
 #
 # We use the weights of this LR classifier to represent the target
 word. Each time
 # we encounter this target word we do a small update step: which can
 be intuitively
 # understood as adjusting the words around the target such that
 they're a little closer
 # together in the embedding space, and a little further away from
 the randomly selected
 # 'negative' words which do not occur in the same context window.
 #
 # For a more rigorous explanation, I highly recommend 'Speech and
 Language Processing',
 # the relevant chapter can be found here:
 # https://web.stanford.edu/~jurafsky/slp3/6.pdf
 for i, line in enumerate(corpus):
 for j, target_word in enumerate(line):
 # Select context words.
 left_context = line[max(j-window_size, 0) : j]
 right_context = line[j+1: (j+1 + window_size)]
 positive_words = left_context + [target_word] + right_context
 # For simplicity, assume that the probability of sampling a context
 / target word is 0.
 # It's really (1 + window_size) / |V|.
 #
 # The skip-gram algorithm does a weighted sample with upweighting of
 rarer words.
 # For simplicity we take an unweighted sample of the entire
 vocabulary.
 negative_words = random.sample(vocabulary, len(positive_words) * 
 alpha)
 positive_idxs = [word_to_idx[word] for word in positive_words]
 negative_idxs = [word_to_idx[word] for word in negative_words]
 context_positive = C[:, positive_idxs]
 context_negative = C[:, negative_idxs]
 a = (_sigmoid(context_positive.T @ W) - 1)
 b = (_sigmoid(context_negative.T @ W))
 # Compute the partial derivative of the classifier loss w.r.t
 # the *context* parameters of the words in each of the two classes.
 dL_by_d_context_positive = a @ W.T
 dL_by_d_context_negative = b @ W.T
 # Compute the partial derivative of the classifier loss w.r.t
 # the *target* word parameters.
 dL_by_dw = context_positive @ a + context_negative @ b
 context_positive_update = e * dL_by_d_context_positive
 context_negative_update = e * dL_by_d_context_negative
 target_word_update = e * dL_by_dw
 # Update the context embedding matrix:
 # move positive words closer to the target word, and negative
 # words further away.
 C[:, positive_idxs] -= context_positive_update.T
 C[:, negative_idxs] -= context_negative_update.T
 # Move the target word's vector closer to the positive context
 # vectors, and further from the negative context vectors.
 W -= target_word_update
 if i % 100 == 0:
 # Log training progress with an example 'king' and 'queen'
 print(f"line {i} of {len(corpus)}")
 rey = W[:, word_to_idx["rey"]]
 reina = W[:, word_to_idx["reina"]]
 print(f"Dot product of 'rey' and 'reina' {np.dot(rey, reina):.3f}")
 print(f"Cosine similarity between 'rey' and 'reina' {
 _cosine_similarity(rey, reina):.3f}")
 # The final embedding matrix can either be `W`, or the sum of `W`
 and `C`.
 # Let's use the latter - why not -_(tsu)_/-
 embedding_matrix = W + C
 return EmbeddingDict(embedding_matrix, word_to_idx)
 if __name__ == "__main__":
 file = Path("data//spanish_billion_words_00")
 word2vec = train_skip_gram(file)
 print("Dot product between 'boy' and 'girl': %s", word2vec["hijo"] *
 word2vec["hija"])
 print("Dot product between 'king' and 'queen': %s", word2vec["rey"] 
 * word2vec["reina"])
 boy_to_girl = word2vec["hijo"] - word2vec["hija"]
 king_to_queen = word2vec["rey"] - word2vec["reina"]
 print("Similarity between the vectors boy->girl and king->queen %s",
 boy_to_girl * king_to_queen)

  * Copy lines
  * Copy permalink
  * View git blame
  * Reference in new issue

[                    ] Go
Footer

 (c) 2023 GitHub, Inc.

Footer navigation

  * Terms
  * Privacy
  * Security
  * Status
  * Docs
  * Contact GitHub
  * Pricing
  * API
  * Training
  * Blog
  * About

You can't perform that action at this time.
You signed in with another tab or window. Reload to refresh your
session. You signed out in another tab or window. Reload to refresh
your session.