The Loyola AI Club's early Spring Semester project, led by Eric Spencer: a sign language interpreter. The original goal was to download a large set of sign language videos from an existing dataset and find out whether a working interpreter could be built from them with OpenCV and Python.
The process was as follows:
def download_youtube_video(url, videos_folder=VIDEOS_FOLDER):
"""Download a YouTube video using yt-dlp and return the local file path."""
video_id = get_video_id(url)
filename = f"{video_id}.mp4"
video_path = os.path.join(videos_folder, filename)
if os.path.exists(video_path):
print(f"[INFO] Video already exists: {video_path}")
return video_path
try:
print(f"[INFO] Downloading video {url} ...")
command = [
"yt-dlp",
"--no-check-certificate",
"-f", "mp4",
"-o", video_path,
url
]
result = subprocess.run(command, check=False, capture_output=True, text=True)
if result.returncode != 0:
print(f"[ERROR] yt-dlp failed: {result.stderr}")
return None
print(f"[INFO] Video saved as {video_path}")
return video_path
except Exception as e:
print(f"[ERROR] Failed to download {url}: {e}")
return None
def main():
# Load data from processed NPZ files
X, y = load_npz_data(DATA_FOLDER)
print(f"[INFO] Loaded {len(X)} samples.")
if len(X) == 0:
print("[ERROR] No data loaded. Exiting.")
return
# Build a label-to-index mapping based on the gesture folder names
unique_labels = sorted(list(set(y)))
label_to_index = {label: idx for idx, label in enumerate(unique_labels)}
print(f"[INFO] Found {len(unique_labels)} unique gesture classes: {unique_labels}")
# Convert string labels to integer indices and then to one-hot vectors
y_indices = np.array([label_to_index[label] for label in y])
num_classes = len(unique_labels)
y_cat = to_categorical(y_indices, num_classes)
# Build a simple Conv3D model
model = Sequential([
Conv3D(32, (3, 3, 3), activation="relu", input_shape=(NUM_FRAMES, IMG_SIZE[0], IMG_SIZE[1], 1)),
MaxPooling3D(pool_size=(1, 2, 2)),
Conv3D(64, (3, 3, 3), activation="relu"),
MaxPooling3D(pool_size=(1, 2, 2)),
Dropout(0.3),
Flatten(),
Dense(128, activation="relu"),
Dropout(0.5),
Dense(num_classes, activation="softmax")
])
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
model.summary()
steps_per_epoch = max(1, len(X) // BATCH_SIZE)
# Set up callbacks: checkpointing and early stopping
checkpoint_callback = ModelCheckpoint(
filepath="model_checkpoint.h5",
monitor="loss",
save_best_only=True,
verbose=1
)
early_stopping_callback = EarlyStopping(
monitor="loss",
patience=5,
verbose=1
)
# Train the model using the data generator (with augmentation)
model.fit(
data_generator(X, y_cat, batch_size=BATCH_SIZE, augment=True),
steps_per_epoch=steps_per_epoch,
epochs=EPOCHS,
callbacks=[checkpoint_callback, early_stopping_callback]
)
model.save("asl_model.h5")
print("[INFO] Model saved as asl_model.h5")
# Process the frame with MediaPipe for hand detection
frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
results = hands_detector.process(frame_rgb)
if results.multi_hand_landmarks:
for hand_landmarks in results.multi_hand_landmarks:
mp_drawing.draw_landmarks(frame, hand_landmarks, mp_hands.HAND_CONNECTIONS)
else:
cv2.putText(frame, "No hand detected", (10, 90),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
The approach ran into problems. The data was stored in .npz format, which suited storage but not training. The video downloads also put strain on a home wifi network that had been assumed to be unlimited, and it turns out a wifi plan can run out. Loyola's network handled the rest of the data transfer.
The second approach used mediapipe to overlay the original videos and transform them into csv dot arrays. It looked promising but did not work, and given the time needed to download and train, the app fell back to something simpler.
What ships uses an existing single-letter ASL interpreter model. It handles 26 letters and one hand.
def preprocess_hand_image(hand_img):
"""Preprocess the hand image for the model"""
if hand_img.size == 0:
return None
# Convert to grayscale and resize
hand_gray = cv2.cvtColor(hand_img, cv2.COLOR_BGR2GRAY)
# Apply adaptive histogram equalization for better contrast
clahe = cv2.createCLAHE(clipLimit=2.0, tileGridSize=(8,8))
hand_eq = clahe.apply(hand_gray)
# Resize to model input size
hand_resized = cv2.resize(hand_eq, (28, 28))
# Normalize and reshape for model input
processed = hand_resized.astype('float32') / 255.0
processed = np.expand_dims(processed, axis=(0, -1))
return processed, hand_resized
The project covered data processing end to end plus a small amount of machine learning.
The layout of this page is copied exactly from the Loyola AI Club's website. See here.