🧤DVC Version Control for Machine Learning

git init

wget https://dvc.org/s3/examples/so/code.zip

unzip code.zip && rm -f code.zip

git add code/

git commit -m "download and initialize code"

mkvirtualenv venv

workon venv

pip install -r code/requirements.txt

dvc init

git commit -m "initialize DVC"

mkdir data

wget -P data https://dvc.org/s3/examples/so/Posts.xml.zip

dvc add data/Posts.xml.zip

git add data/Posts.xml.zip.dvc data/.gitignore

git commit -m "add dataset"

dvc run -d data/Posts.xml.zip ^
        -o data/Posts.xml ^
        -f extract.dvc ^
        unzip data/Posts.xml.zip -d data

dvc run -d code/xml_to_tsv.py -d data/Posts.xml ^
          -o data/Posts.tsv ^
          -f prepare.dvc ^
          python code/xml_to_tsv.py data/Posts.xml data/Posts.tsv

dvc run -d code/split_train_test.py -d data/Posts.tsv ^
          -o data/Posts-train.tsv -o data/Posts-test.tsv ^
          -f split.dvc ^
          python code/split_train_test.py data/Posts.tsv 0.2 20170426 ^
          data/Posts-train.tsv data/Posts-test.tsv

dvc run -d code/featurization.py -d data/Posts-train.tsv -d data/Posts-test.tsv ^
        -o data/matrix-train.pkl -o data/matrix-test.pkl ^
        -f featurize.dvc ^
        python code/featurization.py data/Posts-train.tsv data/Posts-test.tsv ^
        data/matrix-train.pkl data/matrix-test.pkl

dvc run -d code/train_model.py -d data/matrix-train.pkl ^
        -o data/model.pkl ^
        -f train.dvc ^
python code/train_model.py data/matrix-train.pkl 20170426 data/model.pkl

dvc run -d code/evaluate.py -d data/model.pkl -d data/matrix-test.pkl ^
          -M auc.metric ^
          -f evaluate.dvc ^
python code/evaluate.py data/model.pkl data/matrix-test.pkl auc.metric

dvc metrics show

auc.metric: AUC: 0.587951

git add *.dvc auc.metric

git commit -am "create pipeline"

notepad code/featurization.py

bag_of_words = CountVectorizer(stop_words='english',
                               max_features=5000,
                               ngram_range=(1, 2))

dvc repro evaluate.dvc

dvc metrics show -a

git add evaluate.dvc auc.metric

git commit -m "add evaluation step to the pipeline

git tag -a "baseline-experiment" -m "baseline"

dvc pipeline show --ascii train.dvc

dvc pipeline show --ascii train.dvc --commands

dvc pipeline show --ascii train.dvc --outs