Merge pull request #146 from didi/install_with_pip

Install with pip

.gitignore

@@ -113,3 +113,10 @@ dpl/output/
 
 # inner egs
 delta_inner
+
+# setup
+delta_nlp.egg-info/
+wheelhouse
+build
+dist
+py36venv

build_pip_pkg.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+PIP_NAME="delta-nlp"
+
+echo "Uninstall ${PIP_NAME} if exist ..."
+pip3 uninstall -y ${PIP_NAME}
+
+echo "Build binary distribution wheel file ..."
+rm -rf build/ ${PIP_NAME}.egg-info/ dist/
+python3 setup.py bdist_wheel

delta/__init__.py

@@ -13,3 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
+
+import os
+
+
+PACKAGE_ROOT_DIR = os.path.dirname(os.path.abspath(__file__))

delta/data/feat/speech_feature_test.py

@@ -24,6 +24,7 @@
 
 from delta.data.feat import speech_ops
 from delta.data.feat import speech_feature
+from delta import PACKAGE_ROOT_DIR
 
 
 #pylint: disable=too-many-instance-attributes
@@ -39,11 +40,11 @@ def setUp(self):
  self.nfft = 512
  self.feat_type = 'logfbank'
 
- rootpath = Path(os.environ['MAIN_ROOT'])
+ package_root = Path(PACKAGE_ROOT_DIR)
  self.wavfile = str(
-  rootpath.joinpath('delta/data/feat/python_speech_features/english.wav'))
+ package_root.joinpath('data/feat/python_speech_features/english.wav'))
  self.featfile = str(
-  rootpath.joinpath('delta/data/feat/python_speech_features/english.npy'))
+ package_root.joinpath('data/feat/python_speech_features/english.npy'))
 
  def tearDown(self):
  ''' tear down '''

delta/data/feat/speech_ops_test.py

@@ -22,6 +22,7 @@
 
 from delta.data.feat.speech_feature import load_wav
 from delta.data.feat import speech_ops as tffeat
+from delta import PACKAGE_ROOT_DIR
 
 
 class SpeechOpsFeatTest(tf.test.TestCase):
@@ -38,8 +39,8 @@ def setUp(self):
  audio_desired_samples=1000,
  add_delta_deltas=False)
  self.wavpath = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/data/feat/python_speech_features/english.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'data/feat/python_speech_features/english.wav'))
  _, self.audio_true = load_wav(self.wavpath, sr=self.sr_true)
 
  def test_read_wav(self):

delta/data/feat/tf_speech_feature_test.py

@@ -22,18 +22,19 @@
 
 from delta.data.feat.speech_feature import load_wav
 from delta.data.feat import tf_speech_feature as tffeat
+from delta import PACKAGE_ROOT_DIR
 
 
 class SpeechFeatTest(tf.test.TestCase):
  ''' tf.signal feat unittest'''
 
  def setUp(self):
  super().setUp()
- main_root = Path(os.environ['MAIN_ROOT'])
+ package_root = Path(PACKAGE_ROOT_DIR)
  self.params = tffeat.speech_params(sr=8000, bins=40, cmvn=False)
  self.wavpath = str(
- main_root.joinpath(
- 'delta/data/feat/python_speech_features/english.wav'))
+ package_root.joinpath(
+ 'data/feat/python_speech_features/english.wav'))
  self.sr_true, self.audio_true = load_wav(str(self.wavpath), sr=8000)
 
  def test_extract_feature(self):

delta/data/frontend/analyfiltbank_test.py

@@ -17,17 +17,18 @@
 import delta.compat as tf
 import os
 from pathlib import Path
+import numpy as np
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.analyfiltbank import Analyfiltbank
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class Test(tf.test.TestCase):
 
  def test_analyfiltbank(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
 

delta/data/frontend/cepstrum_test.py

@@ -14,21 +14,21 @@
 # limitations under the License.
 # ==============================================================================
 
+import numpy as np
 import delta.compat as tf
-import os
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.cepstrum import Cepstrum
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class CepstrumTest(tf.test.TestCase):
 
  def test_cepstrum(self):
 
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/fbank_pitch_test.py

@@ -19,14 +19,15 @@
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.fbank_pitch import FbankPitch
+from delta import PACKAGE_ROOT_DIR
 
 
 class FbankPitchTest(tf.test.TestCase):
 
  def test_FbankPitch(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/fbank_test.py

@@ -19,14 +19,15 @@
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.fbank import Fbank
+from delta import PACKAGE_ROOT_DIR
 
 
 class FbankTest(tf.test.TestCase):
 
  def test_fbank(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/framepow_test.py

@@ -16,18 +16,19 @@
 
 import delta.compat as tf
 import os
+import numpy as np
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.framepow import Framepow
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class FramepowTest(tf.test.TestCase):
 
  def test_framepow(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/pitch_test.py

@@ -17,18 +17,20 @@
 import delta.compat as tf
 import os
 from pathlib import Path
+import numpy as np
+
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.pitch import Pitch
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class PitchTest(tf.test.TestCase):
 
  def test_pitch(self):
 
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()
  input_data, sample_rate = read_wav.call(wav_path)

delta/data/frontend/plp_test.py

@@ -17,17 +17,18 @@
 import delta.compat as tf
 import os
 from pathlib import Path
+import numpy as np
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.plp import Plp
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class PlpTest(tf.test.TestCase):
 
  def test_plp(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/read_wav_test.py

@@ -15,18 +15,19 @@
 # ==============================================================================
 
 import delta.compat as tf
-import os
 from pathlib import Path
-from delta.data.frontend.read_wav import ReadWav
 import librosa
 
+from delta.data.frontend.read_wav import ReadWav
+from delta import PACKAGE_ROOT_DIR
+
 
 class ReadWavTest(tf.test.TestCase):
 
  def test_read_wav(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params({'sample_rate': 16000.0}).instantiate()

delta/data/frontend/spectrum_test.py

@@ -16,18 +16,19 @@
 
 import delta.compat as tf
 import os
+import numpy as np
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.spectrum import Spectrum
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class SpectrumTest(tf.test.TestCase):
 
  def test_spectrum(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/frontend/synthfiltbank_test.py

@@ -20,14 +20,15 @@
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.analyfiltbank import Analyfiltbank
 from delta.data.frontend.synthfiltbank import Synthfiltbank
+from delta import PACKAGE_ROOT_DIR
 
 
 class Test(tf.test.TestCase):
 
  def test_synthfiltbank(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
 

delta/data/frontend/write_wav_test.py

@@ -19,22 +19,23 @@
 from pathlib import Path
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.write_wav import WriteWav
+from delta import PACKAGE_ROOT_DIR
 
 
 class WriteWavTest(tf.test.TestCase):
 
  def test_write_wav(self):
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False) as sess:
  read_wav = ReadWav.params().instantiate()
  input_data, sample_rate = read_wav(wav_path)
  write_wav = WriteWav.params().instantiate()
  new_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln_new.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln_new.wav'))
  writewav_op = write_wav(new_path, input_data, sample_rate)
  sess.run(writewav_op)
  test_data, test_sample_rate = read_wav(new_path)

delta/data/frontend/zcr_test.py

@@ -17,18 +17,20 @@
 import delta.compat as tf
 import os
 from pathlib import Path
+import numpy as np
+
 from delta.data.frontend.read_wav import ReadWav
 from delta.data.frontend.zcr import Zcr
-import numpy as np
+from delta import PACKAGE_ROOT_DIR
 
 
 class ZcrTest(tf.test.TestCase):
 
  def test_zcr(self):
 
  wav_path = str(
- Path(os.environ['MAIN_ROOT']).joinpath(
- 'delta/layers/ops/data/sm1_cln.wav'))
+ Path(PACKAGE_ROOT_DIR).joinpath(
+ 'layers/ops/data/sm1_cln.wav'))
 
  with self.cached_session(use_gpu=False, force_gpu=False):
  read_wav = ReadWav.params().instantiate()

delta/data/preprocess/base_preparer.py

@@ -138,7 +138,7 @@ def prepare_one_raw_data(self, one_path, one_path_after, mode,
  text_after = [one_line.decode("utf-8") for one_line in text_after_arr]
  all_texts += text_after
  one_text_after = text_after
- self.config['data']['{}_data_size'.format(mode)] = len(one_text_after[0])
+
  one_label_after = []
  if not infer_without_label:
  if self.multi_output:
@@ -161,7 +161,6 @@ def prepare_one_raw_data(self, one_path, one_path_after, mode,
  ]
  all_labels += one_label_after
 
- logging.debug(f"one_text_after: {len(one_text_after)}")
  self.save_a_raw_file(one_label_after, one_text_after, one_path_after,
  infer_without_label)