전이학습을 따라하는데 결과값이 좋게 나오질 않습니다

import tensorflow as tf
from keras.applications import InceptionV3

gpus = tf.config.experimental.list_physical_devices('GPU')

for gpu in gpus:
    tf.config.experimental.set_memory_growth(gpu, True)

dataset = 'P:/Data/dogs-vs-cats-redux-kernels-edition/train/dataset/'

train_ds = tf.keras.preprocessing.image_dataset_from_directory(
    dataset,
    image_size=(150, 150),
    batch_size=64,
    subset='training',
    validation_split=0.2,
    seed=1234,
)

val_ds = tf.keras.preprocessing.image_dataset_from_directory(
    dataset,
    image_size=(150, 150),
    batch_size=64,
    subset='validation',
    validation_split=0.2,
    seed=1234,
)

inception_model = InceptionV3(input_shape=(150,150,3), include_top=False, weights = None)
inception_model.load_weights('inception_v3.h5')

# inception_model.summary()

for i in inception_model.layers :
    i.trainable = False
    
unfreeze = False
for i in inception_model.layers :
    if i.name == 'mixed6' :
        unfreeze= True
    if unfreeze == True :
        i.trainable = True
        
    
마지막레이어 = inception_model.get_layer('mixed7')

layer1 = tf.keras.layers.Flatten()(마지막레이어.output)
layer2 = tf.keras.layers.Dense(1024,activation='relu')(layer1)
drop1 = tf.keras.layers.Dropout(0.2)(layer2)
layer3 = tf.keras.layers.Dense(1,activation='sigmoid')(drop1)

model = tf.keras.Model(
    inception_model.input,
    layer3
)

model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['acc'])

model.fit(train_ds, validation_data=val_ds, epochs=2)

model.compile(loss='binary_crossentropy', optimizer=tf.keras.optimizers.Adam(lr=0.00001), metrics=['acc'])

model.fit(train_ds, validation_data=val_ds, epochs=2)

# print(마지막레이어)
# print(마지막레이어.output)
# print(마지막레이어.output_shape)



영상 보면서 코드 그대로 따라치면서 하는데

오히려 accuracy가 떨어지는 현상이 일어나는데 이유가 뭘까요?