@online{huang2024tokenization,
  author       = {Huang, Xuanqiang Angelo},
  title        = {Tokenization},
  date         = {2024-01-20},
  organization = {Xuanqiang Angelo Huang's Blog},
  url          = {https://flecart.github.io/notes/tokenization/},
  langid       = {english},
  abstract     = {Introduction to tokenization \# Tokenization is the process of converting normal strings into small little pieces that could be fed into one of our models. It usually comes from a tradition in programming languages, as we can see in Automi e Regexp where we define a specific token}
}